Add ckan patch script and patches as part of the ckan image build process

2020-09-20 00:03:36 +02:00 · 2020-09-20 00:03:36 +02:00 · 4e0634fca7
parent bb3e1d401c
commit 4e0634fca7
19 changed files with 105 additions and 8 deletions
--- a/compose/.ckan-env
+++ b/compose/.ckan-env
@ -23,3 +23,7 @@ CKAN_SMTP_STARTTLS=True
 CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
--- a/compose/.env
+++ b/compose/.env
@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
 DATAPUSHER_CHUNK_SIZE=16384
 DATAPUSHER_CHUNK_INSERT_ROWS=250
 DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000

 # Redis
 REDIS_VERSION=6.0.7
--- a/compose/docker-compose.yml
+++ b/compose/docker-compose.yml
@ -25,7 +25,6 @@ services:
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -46,6 +45,10 @@ services:
      - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
      - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
      - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
+      - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
+

  db:
    container_name: db
--- a/examples/harvest/.ckan-env
+++ b/examples/harvest/.ckan-env
@ -24,6 +24,10 @@ CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost

+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
+
 # Harvest settings
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
--- a/examples/harvest/.env
+++ b/examples/harvest/.env
@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
 DATAPUSHER_CHUNK_SIZE=16384
 DATAPUSHER_CHUNK_INSERT_ROWS=250
 DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000

 # Redis
 REDIS_VERSION=6.0.7
--- a/examples/harvest/docker-compose.yml
+++ b/examples/harvest/docker-compose.yml
@ -26,7 +26,6 @@ services:
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -54,7 +53,6 @@ services:
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -80,7 +78,6 @@ services:
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -105,7 +102,6 @@ services:
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -130,7 +126,6 @@ services:
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -149,6 +144,9 @@ services:
      - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
      - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
      - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
+      - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}

  db:
    container_name: db
--- a/examples/s3filestore/.ckan-env
+++ b/examples/s3filestore/.ckan-env
@ -24,6 +24,10 @@ CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost

+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
+
 # S3/MINIO settings
 CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=MINIOACCESSKEY
 CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=MINIOSECRETKEY
--- a/examples/s3filestore/.env
+++ b/examples/s3filestore/.env
@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
 DATAPUSHER_CHUNK_SIZE=16384
 DATAPUSHER_CHUNK_INSERT_ROWS=250
 DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000

 # Redis
 REDIS_VERSION=6.0.7
--- a/examples/s3filestore/docker-compose.yml
+++ b/examples/s3filestore/docker-compose.yml
@ -26,7 +26,6 @@ services:
      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
      - CKAN_SITE_URL=${CKAN_SITE_URL}
      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -45,6 +44,9 @@ services:
      - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
      - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
      - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
+      - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}

  db:
    container_name: db
--- a/images/ckan/2.7/Dockerfile
+++ b/images/ckan/2.7/Dockerfile
@ -26,6 +26,7 @@ RUN apk add --no-cache \
        autoconf \
        automake \
        libtool \
+        patch \
        musl-dev \
        pcre-dev \
        pcre \
@ -47,6 +48,11 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \

 # Fetch and build CKAN and requirements
 RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+RUN ${SRC_DIR}/apply_ckan_patches.sh
 RUN rm -rf /srv/app/src/ckan/.git
 RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
 RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2
--- a/images/ckan/2.7/patches/00_datapusher_fix_callback_url.patch
+++ b/images/ckan/2.7/patches/00_datapusher_fix_callback_url.patch
@ -0,0 +1,18 @@
+--- ckan/ckanext/datapusher/logic/action.py	2020-09-19 21:25:06.525350202 +0200
+++ action_fix_callback_url.py	2020-09-19 21:27:43.596501963 +0200
+@@ -61,13 +61,13 @@
+ 
+     datapusher_url = config.get('ckan.datapusher.url')
+ 
+-    site_url = h.url_for('/', qualified=True)
+-
+     callback_url_base = config.get('ckan.datapusher.callback_url_base')
+     if callback_url_base:
+        site_url = callback_url_base
+         callback_url = urlparse.urljoin(
+             callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook')
+     else:
+        site_url = h.url_for('/', qualified=True)
+         callback_url = h.url_for(
+             '/api/3/action/datapusher_hook', qualified=True)
+ 
--- a/images/ckan/2.7/scripts/apply_ckan_patches.sh
+++ b/images/ckan/2.7/scripts/apply_ckan_patches.sh
@ -0,0 +1,5 @@
+#!/bin/bash
+shopt -s nullglob
+for patch in patches/*.patch; do
+    /usr/bin/patch -p0 -i $patch
+done
--- a/images/ckan/2.8/Dockerfile
+++ b/images/ckan/2.8/Dockerfile
@ -25,6 +25,7 @@ RUN apk add --no-cache \
        g++ \
        autoconf \
        automake \
+        patch \
        libtool \
        musl-dev \
        pcre-dev \
@ -43,6 +44,11 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \

 # Fetch and build CKAN and requirements
 RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+RUN ${SRC_DIR}/apply_ckan_patches.sh
 RUN rm -rf /srv/app/src/ckan/.git
 RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
 RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2
--- a/images/ckan/2.8/patches/00_datapusher_fix_callback_url.patch
+++ b/images/ckan/2.8/patches/00_datapusher_fix_callback_url.patch
@ -0,0 +1,18 @@
+--- ckan/ckanext/datapusher/logic/action.py	2020-09-19 21:25:06.525350202 +0200
+++ action_fix_callback_url.py	2020-09-19 21:27:43.596501963 +0200
+@@ -61,13 +61,13 @@
+ 
+     datapusher_url = config.get('ckan.datapusher.url')
+ 
+-    site_url = h.url_for('/', qualified=True)
+-
+     callback_url_base = config.get('ckan.datapusher.callback_url_base')
+     if callback_url_base:
+        site_url = callback_url_base
+         callback_url = urlparse.urljoin(
+             callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook')
+     else:
+        site_url = h.url_for('/', qualified=True)
+         callback_url = h.url_for(
+             '/api/3/action/datapusher_hook', qualified=True)
+ 
--- a/images/ckan/2.8/scripts/apply_ckan_patches.sh
+++ b/images/ckan/2.8/scripts/apply_ckan_patches.sh
@ -0,0 +1,5 @@
+#!/bin/bash
+shopt -s nullglob
+for patch in patches/*.patch; do
+    /usr/bin/patch -p0 -i $patch
+done
--- a/images/ckan/2.9/Dockerfile
+++ b/images/ckan/2.9/Dockerfile
@ -26,6 +26,7 @@ RUN apk add --no-cache \
        autoconf \
        automake \
        libtool \
+        patch \
        musl-dev \
        pcre-dev \
        pcre \
@ -49,6 +50,11 @@ RUN pip install setuptools==44.1.0

 # Fetch and build CKAN and requirements
 RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+RUN ${SRC_DIR}/apply_ckan_patches.sh
 RUN rm -rf /srv/app/src/ckan/.git
 RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
 RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2
--- a/images/ckan/2.9/scripts/apply_ckan_patches.sh
+++ b/images/ckan/2.9/scripts/apply_ckan_patches.sh
@ -0,0 +1,5 @@
+#!/bin/bash
+shopt -s nullglob
+for patch in patches/*.patch; do
+    /usr/bin/patch -p0 -i $patch
+done
--- a/images/datapusher/Dockerfile
+++ b/images/datapusher/Dockerfile
@ -5,7 +5,7 @@ FROM alpine:3.12 as build

 # Set datapusher version to build
 ENV GIT_URL https://github.com/keitaroinc/datapusher.git
-ENV GIT_BRANCH parametrize-job-config
+ENV GIT_BRANCH master
 ENV REQUIREMENTS_URL https://raw.githubusercontent.com/keitaroinc/datapusher/${GIT_BRANCH}/requirements.txt

 # Set src dirs
--- a/images/datapusher/setup/datapusher_settings.py
+++ b/images/datapusher/setup/datapusher_settings.py
@ -31,3 +31,7 @@ MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '102400
 CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384'))
 CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250'))
 DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30'))
+
+# Rewrite resource URL's when ckan callback url base is used
+REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', False)
+REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/')