Add ckan patch script and patches as part of the ckan image build process

compose/.ckan-env

@@ -23,3 +23,7 @@ CKAN_SMTP_STARTTLS=True
 CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/

compose/.env

@@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
 DATAPUSHER_CHUNK_SIZE=16384
 DATAPUSHER_CHUNK_INSERT_ROWS=250
 DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000
 
 # Redis
 REDIS_VERSION=6.0.7

compose/docker-compose.yml

@@ -25,7 +25,6 @@ services:
       - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
       - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
       - CKAN_SITE_URL=${CKAN_SITE_URL}
       - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -46,6 +45,10 @@ services:
       - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
       - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
       - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
+      - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
+
 
   db:
     container_name: db

examples/harvest/.ckan-env

@@ -24,6 +24,10 @@ CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
 
+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
+
 # Harvest settings
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis

examples/harvest/.env

@@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
 DATAPUSHER_CHUNK_SIZE=16384
 DATAPUSHER_CHUNK_INSERT_ROWS=250
 DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000
 
 # Redis
 REDIS_VERSION=6.0.7

examples/harvest/docker-compose.yml

@@ -26,7 +26,6 @@ services:
       - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
       - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
       - CKAN_SITE_URL=${CKAN_SITE_URL}
       - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -54,7 +53,6 @@ services:
       - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
       - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
       - CKAN_SITE_URL=${CKAN_SITE_URL}
       - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -80,7 +78,6 @@ services:
       - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
       - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
       - CKAN_SITE_URL=${CKAN_SITE_URL}
       - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -105,7 +102,6 @@ services:
       - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
       - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
       - CKAN_SITE_URL=${CKAN_SITE_URL}
       - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -130,7 +126,6 @@ services:
       - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
       - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
       - CKAN_SITE_URL=${CKAN_SITE_URL}
       - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -149,6 +144,9 @@ services:
       - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
       - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
       - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
+      - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
 
   db:
     container_name: db

examples/s3filestore/.ckan-env

@@ -24,6 +24,10 @@ CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
 
+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
+
 # S3/MINIO settings
 CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=MINIOACCESSKEY
 CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=MINIOSECRETKEY

examples/s3filestore/.env

@@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
 DATAPUSHER_CHUNK_SIZE=16384
 DATAPUSHER_CHUNK_INSERT_ROWS=250
 DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000
 
 # Redis
 REDIS_VERSION=6.0.7

examples/s3filestore/docker-compose.yml

@@ -26,7 +26,6 @@ services:
       - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
       - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8000
       - CKAN_SITE_URL=${CKAN_SITE_URL}
       - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -45,6 +44,9 @@ services:
       - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
       - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
       - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
+      - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
 
   db:
     container_name: db

images/ckan/2.7/Dockerfile

@@ -26,6 +26,7 @@ RUN apk add --no-cache \
         autoconf \
         automake \
         libtool \
+        patch \
         musl-dev \
         pcre-dev \
         pcre \
@@ -47,6 +48,11 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
 
 # Fetch and build CKAN and requirements
 RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+RUN ${SRC_DIR}/apply_ckan_patches.sh
 RUN rm -rf /srv/app/src/ckan/.git
 RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
 RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2

images/ckan/2.7/patches/00_datapusher_fix_callback_url.patch

@@ -0,0 +1,18 @@
+--- ckan/ckanext/datapusher/logic/action.py	2020-09-19 21:25:06.525350202 +0200
++++ action_fix_callback_url.py	2020-09-19 21:27:43.596501963 +0200
+@@ -61,13 +61,13 @@
+ 
+     datapusher_url = config.get('ckan.datapusher.url')
+ 
+-    site_url = h.url_for('/', qualified=True)
+-
+     callback_url_base = config.get('ckan.datapusher.callback_url_base')
+     if callback_url_base:
++        site_url = callback_url_base
+         callback_url = urlparse.urljoin(
+             callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook')
+     else:
++        site_url = h.url_for('/', qualified=True)
+         callback_url = h.url_for(
+             '/api/3/action/datapusher_hook', qualified=True)
+ 

images/ckan/2.7/scripts/apply_ckan_patches.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+shopt -s nullglob
+for patch in patches/*.patch; do
+    /usr/bin/patch -p0 -i $patch
+done

images/ckan/2.8/Dockerfile

@@ -25,6 +25,7 @@ RUN apk add --no-cache \
         g++ \
         autoconf \
         automake \
+        patch \
         libtool \
         musl-dev \
         pcre-dev \
@@ -43,6 +44,11 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
 
 # Fetch and build CKAN and requirements
 RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+RUN ${SRC_DIR}/apply_ckan_patches.sh
 RUN rm -rf /srv/app/src/ckan/.git
 RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
 RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2

images/ckan/2.8/patches/00_datapusher_fix_callback_url.patch

@@ -0,0 +1,18 @@
+--- ckan/ckanext/datapusher/logic/action.py	2020-09-19 21:25:06.525350202 +0200
++++ action_fix_callback_url.py	2020-09-19 21:27:43.596501963 +0200
+@@ -61,13 +61,13 @@
+ 
+     datapusher_url = config.get('ckan.datapusher.url')
+ 
+-    site_url = h.url_for('/', qualified=True)
+-
+     callback_url_base = config.get('ckan.datapusher.callback_url_base')
+     if callback_url_base:
++        site_url = callback_url_base
+         callback_url = urlparse.urljoin(
+             callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook')
+     else:
++        site_url = h.url_for('/', qualified=True)
+         callback_url = h.url_for(
+             '/api/3/action/datapusher_hook', qualified=True)
+ 

images/ckan/2.8/scripts/apply_ckan_patches.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+shopt -s nullglob
+for patch in patches/*.patch; do
+    /usr/bin/patch -p0 -i $patch
+done

images/ckan/2.9/Dockerfile

@@ -26,6 +26,7 @@ RUN apk add --no-cache \
         autoconf \
         automake \
         libtool \
+        patch \
         musl-dev \
         pcre-dev \
         pcre \
@@ -49,6 +50,11 @@ RUN pip install setuptools==44.1.0
 
 # Fetch and build CKAN and requirements
 RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+RUN ${SRC_DIR}/apply_ckan_patches.sh
 RUN rm -rf /srv/app/src/ckan/.git
 RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
 RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2

images/ckan/2.9/scripts/apply_ckan_patches.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+shopt -s nullglob
+for patch in patches/*.patch; do
+    /usr/bin/patch -p0 -i $patch
+done

images/datapusher/Dockerfile

@@ -5,7 +5,7 @@ FROM alpine:3.12 as build
 
 # Set datapusher version to build
 ENV GIT_URL https://github.com/keitaroinc/datapusher.git
-ENV GIT_BRANCH parametrize-job-config
+ENV GIT_BRANCH master
 ENV REQUIREMENTS_URL https://raw.githubusercontent.com/keitaroinc/datapusher/${GIT_BRANCH}/requirements.txt
 
 # Set src dirs

images/datapusher/setup/datapusher_settings.py

@@ -31,3 +31,7 @@ MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '102400
 CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384'))
 CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250'))
 DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30'))
+
+# Rewrite resource URL's when ckan callback url base is used
+REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', False)
+REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/')