diff --git a/compose/.ckan-env b/compose/.ckan-env index 344ee7d..59e99bd 100644 --- a/compose/.ckan-env +++ b/compose/.ckan-env @@ -23,3 +23,7 @@ CKAN_SMTP_STARTTLS=True CKAN_SMTP_USER=user CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost + +# Datapusher configuration +CKAN__DATAPUSHER__URL=http://datapusher:8000 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ diff --git a/compose/.env b/compose/.env index 89b30ec..a8a815e 100644 --- a/compose/.env +++ b/compose/.env @@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760 DATAPUSHER_CHUNK_SIZE=16384 DATAPUSHER_CHUNK_INSERT_ROWS=250 DATAPUSHER_DOWNLOAD_TIMEOUT=30 +DATAPUSHER_SSL_VERIFY=False +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 # Redis REDIS_VERSION=6.0.7 diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index f8f48c4..675fbf8 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -25,7 +25,6 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -46,6 +45,10 @@ services: - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} + - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY} + - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} + - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} + db: container_name: db diff --git a/examples/harvest/.ckan-env b/examples/harvest/.ckan-env index c1a0993..3801b91 100644 --- a/examples/harvest/.ckan-env +++ b/examples/harvest/.ckan-env @@ -24,6 +24,10 @@ CKAN_SMTP_USER=user CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost +# Datapusher configuration +CKAN__DATAPUSHER__URL=http://datapusher:8000 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ + # Harvest settings CKAN__HARVEST__MQ__TYPE=redis CKAN__HARVEST__MQ__HOSTNAME=redis diff --git a/examples/harvest/.env b/examples/harvest/.env index 89b30ec..a8a815e 100644 --- a/examples/harvest/.env +++ b/examples/harvest/.env @@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760 DATAPUSHER_CHUNK_SIZE=16384 DATAPUSHER_CHUNK_INSERT_ROWS=250 DATAPUSHER_DOWNLOAD_TIMEOUT=30 +DATAPUSHER_SSL_VERIFY=False +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 # Redis REDIS_VERSION=6.0.7 diff --git a/examples/harvest/docker-compose.yml b/examples/harvest/docker-compose.yml index 2d55864..fc2f961 100644 --- a/examples/harvest/docker-compose.yml +++ b/examples/harvest/docker-compose.yml @@ -26,7 +26,6 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -54,7 +53,6 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -80,7 +78,6 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -105,7 +102,6 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -130,7 +126,6 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -149,6 +144,9 @@ services: - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} + - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY} + - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} + - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} db: container_name: db diff --git a/examples/s3filestore/.ckan-env b/examples/s3filestore/.ckan-env index f3359ae..95e1820 100644 --- a/examples/s3filestore/.ckan-env +++ b/examples/s3filestore/.ckan-env @@ -24,6 +24,10 @@ CKAN_SMTP_USER=user CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost +# Datapusher configuration +CKAN__DATAPUSHER__URL=http://datapusher:8000 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ + # S3/MINIO settings CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=MINIOACCESSKEY CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=MINIOSECRETKEY diff --git a/examples/s3filestore/.env b/examples/s3filestore/.env index 4068526..542a31a 100644 --- a/examples/s3filestore/.env +++ b/examples/s3filestore/.env @@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760 DATAPUSHER_CHUNK_SIZE=16384 DATAPUSHER_CHUNK_INSERT_ROWS=250 DATAPUSHER_DOWNLOAD_TIMEOUT=30 +DATAPUSHER_SSL_VERIFY=False +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 # Redis REDIS_VERSION=6.0.7 diff --git a/examples/s3filestore/docker-compose.yml b/examples/s3filestore/docker-compose.yml index d195fbb..a2eb487 100644 --- a/examples/s3filestore/docker-compose.yml +++ b/examples/s3filestore/docker-compose.yml @@ -26,7 +26,6 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -45,6 +44,9 @@ services: - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} + - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY} + - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} + - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} db: container_name: db diff --git a/images/ckan/2.7/Dockerfile b/images/ckan/2.7/Dockerfile index 74878a7..ffdcd9f 100644 --- a/images/ckan/2.7/Dockerfile +++ b/images/ckan/2.7/Dockerfile @@ -26,6 +26,7 @@ RUN apk add --no-cache \ autoconf \ automake \ libtool \ + patch \ musl-dev \ pcre-dev \ pcre \ @@ -47,6 +48,11 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ # Fetch and build CKAN and requirements RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +RUN ${SRC_DIR}/apply_ckan_patches.sh RUN rm -rf /srv/app/src/ckan/.git RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2 diff --git a/images/ckan/2.7/patches/00_datapusher_fix_callback_url.patch b/images/ckan/2.7/patches/00_datapusher_fix_callback_url.patch new file mode 100644 index 0000000..83d03d5 --- /dev/null +++ b/images/ckan/2.7/patches/00_datapusher_fix_callback_url.patch @@ -0,0 +1,18 @@ +--- ckan/ckanext/datapusher/logic/action.py 2020-09-19 21:25:06.525350202 +0200 ++++ action_fix_callback_url.py 2020-09-19 21:27:43.596501963 +0200 +@@ -61,13 +61,13 @@ + + datapusher_url = config.get('ckan.datapusher.url') + +- site_url = h.url_for('/', qualified=True) +- + callback_url_base = config.get('ckan.datapusher.callback_url_base') + if callback_url_base: ++ site_url = callback_url_base + callback_url = urlparse.urljoin( + callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook') + else: ++ site_url = h.url_for('/', qualified=True) + callback_url = h.url_for( + '/api/3/action/datapusher_hook', qualified=True) + diff --git a/images/ckan/2.7/scripts/apply_ckan_patches.sh b/images/ckan/2.7/scripts/apply_ckan_patches.sh new file mode 100755 index 0000000..a7bceb9 --- /dev/null +++ b/images/ckan/2.7/scripts/apply_ckan_patches.sh @@ -0,0 +1,5 @@ +#!/bin/bash +shopt -s nullglob +for patch in patches/*.patch; do + /usr/bin/patch -p0 -i $patch +done diff --git a/images/ckan/2.8/Dockerfile b/images/ckan/2.8/Dockerfile index ad29d5d..028a097 100644 --- a/images/ckan/2.8/Dockerfile +++ b/images/ckan/2.8/Dockerfile @@ -25,6 +25,7 @@ RUN apk add --no-cache \ g++ \ autoconf \ automake \ + patch \ libtool \ musl-dev \ pcre-dev \ @@ -43,6 +44,11 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ # Fetch and build CKAN and requirements RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +RUN ${SRC_DIR}/apply_ckan_patches.sh RUN rm -rf /srv/app/src/ckan/.git RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2 diff --git a/images/ckan/2.8/patches/00_datapusher_fix_callback_url.patch b/images/ckan/2.8/patches/00_datapusher_fix_callback_url.patch new file mode 100644 index 0000000..83d03d5 --- /dev/null +++ b/images/ckan/2.8/patches/00_datapusher_fix_callback_url.patch @@ -0,0 +1,18 @@ +--- ckan/ckanext/datapusher/logic/action.py 2020-09-19 21:25:06.525350202 +0200 ++++ action_fix_callback_url.py 2020-09-19 21:27:43.596501963 +0200 +@@ -61,13 +61,13 @@ + + datapusher_url = config.get('ckan.datapusher.url') + +- site_url = h.url_for('/', qualified=True) +- + callback_url_base = config.get('ckan.datapusher.callback_url_base') + if callback_url_base: ++ site_url = callback_url_base + callback_url = urlparse.urljoin( + callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook') + else: ++ site_url = h.url_for('/', qualified=True) + callback_url = h.url_for( + '/api/3/action/datapusher_hook', qualified=True) + diff --git a/images/ckan/2.8/scripts/apply_ckan_patches.sh b/images/ckan/2.8/scripts/apply_ckan_patches.sh new file mode 100755 index 0000000..a7bceb9 --- /dev/null +++ b/images/ckan/2.8/scripts/apply_ckan_patches.sh @@ -0,0 +1,5 @@ +#!/bin/bash +shopt -s nullglob +for patch in patches/*.patch; do + /usr/bin/patch -p0 -i $patch +done diff --git a/images/ckan/2.9/Dockerfile b/images/ckan/2.9/Dockerfile index c65125b..c4f2dc9 100644 --- a/images/ckan/2.9/Dockerfile +++ b/images/ckan/2.9/Dockerfile @@ -26,6 +26,7 @@ RUN apk add --no-cache \ autoconf \ automake \ libtool \ + patch \ musl-dev \ pcre-dev \ pcre \ @@ -49,6 +50,11 @@ RUN pip install setuptools==44.1.0 # Fetch and build CKAN and requirements RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +RUN ${SRC_DIR}/apply_ckan_patches.sh RUN rm -rf /srv/app/src/ckan/.git RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2 diff --git a/images/ckan/2.9/scripts/apply_ckan_patches.sh b/images/ckan/2.9/scripts/apply_ckan_patches.sh new file mode 100755 index 0000000..a7bceb9 --- /dev/null +++ b/images/ckan/2.9/scripts/apply_ckan_patches.sh @@ -0,0 +1,5 @@ +#!/bin/bash +shopt -s nullglob +for patch in patches/*.patch; do + /usr/bin/patch -p0 -i $patch +done diff --git a/images/datapusher/Dockerfile b/images/datapusher/Dockerfile index 5bff8c4..1428b2d 100644 --- a/images/datapusher/Dockerfile +++ b/images/datapusher/Dockerfile @@ -5,7 +5,7 @@ FROM alpine:3.12 as build # Set datapusher version to build ENV GIT_URL https://github.com/keitaroinc/datapusher.git -ENV GIT_BRANCH parametrize-job-config +ENV GIT_BRANCH master ENV REQUIREMENTS_URL https://raw.githubusercontent.com/keitaroinc/datapusher/${GIT_BRANCH}/requirements.txt # Set src dirs diff --git a/images/datapusher/setup/datapusher_settings.py b/images/datapusher/setup/datapusher_settings.py index cf0a138..17e267c 100644 --- a/images/datapusher/setup/datapusher_settings.py +++ b/images/datapusher/setup/datapusher_settings.py @@ -31,3 +31,7 @@ MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '102400 CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384')) CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250')) DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30')) + +# Rewrite resource URL's when ckan callback url base is used +REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', False) +REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/')