Merge pull request #26 from keitaroinc/add-after-prerun-point

Add after prerun entrypoint
This commit is contained in:
Благоја Стојкоски 2020-09-21 13:16:00 +02:00 committed by GitHub
commit 240f1c213c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 159 additions and 12 deletions

View File

@ -73,8 +73,10 @@ RUN rm -rf /srv/app/ext_wheels
USER ckan
```
### Adding prerun scripts
You can add scripts to CKAN custom images and copy them to the *docker-entrypoint.d* directory. Any *.sh or *.py file in that directory will be executed after the main initialization script (prerun.py) is executed.
### Adding init and afterinit scripts
You can add scripts to CKAN custom images and copy them to the *docker-entrypoint.d* directory. Any *.sh or *.py file in that directory will be executed before the main initialization script (prerun.py) is executed.
You can add scripts to CKAN custom images and copy them to the *docker-afterinit.d* directory. Any *.sh or *.py file in that directory will be executed after the main initialization script (prerun.py) is executed.
## Build
To build a CKAN image run:

View File

@ -23,3 +23,7 @@ CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
# Datapusher configuration
CKAN__DATAPUSHER__URL=http://datapusher:8000
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/

View File

@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
DATAPUSHER_CHUNK_SIZE=16384
DATAPUSHER_CHUNK_INSERT_ROWS=250
DATAPUSHER_DOWNLOAD_TIMEOUT=30
DATAPUSHER_SSL_VERIFY=False
DATAPUSHER_REWRITE_RESOURCES=True
DATAPUSHER_REWRITE_URL=http://ckan:5000
# Redis
REDIS_VERSION=6.0.7

View File

@ -25,7 +25,6 @@ services:
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_DATAPUSHER_URL=http://datapusher:8000
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -46,6 +45,10 @@ services:
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
- DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
db:
container_name: db

View File

@ -24,6 +24,10 @@ CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
# Datapusher configuration
CKAN__DATAPUSHER__URL=http://datapusher:8000
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
# Harvest settings
CKAN__HARVEST__MQ__TYPE=redis
CKAN__HARVEST__MQ__HOSTNAME=redis

View File

@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
DATAPUSHER_CHUNK_SIZE=16384
DATAPUSHER_CHUNK_INSERT_ROWS=250
DATAPUSHER_DOWNLOAD_TIMEOUT=30
DATAPUSHER_SSL_VERIFY=False
DATAPUSHER_REWRITE_RESOURCES=True
DATAPUSHER_REWRITE_URL=http://ckan:5000
# Redis
REDIS_VERSION=6.0.7

View File

@ -48,8 +48,8 @@ RUN pip install --no-index --find-links=/srv/app/ext_wheels ckanext-harvest && \
# Remove wheels
RUN rm -rf /srv/app/ext_wheels
# Add harvest entrypoint script
COPY ./scripts/00_harvest.sh ${APP_DIR}/docker-entrypoint.d/00_harvest.sh
# Add harvest afterinit script
COPY ./afterinit.d/00_harvest.sh ${APP_DIR}/docker-afterinit.d/00_harvest.sh
# Switch to the ckan user
USER ckan

View File

@ -26,7 +26,6 @@ services:
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_DATAPUSHER_URL=http://datapusher:8000
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -54,7 +53,6 @@ services:
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_DATAPUSHER_URL=http://datapusher:8000
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -80,7 +78,6 @@ services:
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_DATAPUSHER_URL=http://datapusher:8000
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -105,7 +102,6 @@ services:
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_DATAPUSHER_URL=http://datapusher:8000
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -130,7 +126,6 @@ services:
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_DATAPUSHER_URL=http://datapusher:8000
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -149,6 +144,9 @@ services:
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
- DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
db:
container_name: db

View File

@ -24,6 +24,10 @@ CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
# Datapusher configuration
CKAN__DATAPUSHER__URL=http://datapusher:8000
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
# S3/MINIO settings
CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=MINIOACCESSKEY
CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=MINIOSECRETKEY

View File

@ -24,6 +24,9 @@ DATAPUSHER_MAX_CONTENT_LENGTH=10485760
DATAPUSHER_CHUNK_SIZE=16384
DATAPUSHER_CHUNK_INSERT_ROWS=250
DATAPUSHER_DOWNLOAD_TIMEOUT=30
DATAPUSHER_SSL_VERIFY=False
DATAPUSHER_REWRITE_RESOURCES=True
DATAPUSHER_REWRITE_URL=http://ckan:5000
# Redis
REDIS_VERSION=6.0.7

View File

@ -26,7 +26,6 @@ services:
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_DATAPUSHER_URL=http://datapusher:8000
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -45,6 +44,9 @@ services:
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
- DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
db:
container_name: db

View File

@ -26,6 +26,7 @@ RUN apk add --no-cache \
autoconf \
automake \
libtool \
patch \
musl-dev \
pcre-dev \
pcre \
@ -47,6 +48,11 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2
@ -161,6 +167,9 @@ COPY setup/app ${APP_DIR}
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1

View File

@ -0,0 +1,18 @@
--- ckan/ckanext/datapusher/logic/action.py 2020-09-19 21:25:06.525350202 +0200
+++ action_fix_callback_url.py 2020-09-19 21:27:43.596501963 +0200
@@ -61,13 +61,13 @@
datapusher_url = config.get('ckan.datapusher.url')
- site_url = h.url_for('/', qualified=True)
-
callback_url_base = config.get('ckan.datapusher.callback_url_base')
if callback_url_base:
+ site_url = callback_url_base
callback_url = urlparse.urljoin(
callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook')
else:
+ site_url = h.url_for('/', qualified=True)
callback_url = h.url_for(
'/api/3/action/datapusher_hook', qualified=True)

View File

@ -0,0 +1,5 @@
#!/bin/bash
shopt -s nullglob
for patch in patches/*.patch; do
/usr/bin/patch -p0 -i $patch
done

View File

@ -18,6 +18,19 @@ UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --e
# Run the prerun script to init CKAN and create the default admin user
python prerun.py
# Run any after prerun/init scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
then
for f in ${APP_DIR}/docker-afterinit.d/*; do
case "$f" in
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then

View File

@ -25,6 +25,7 @@ RUN apk add --no-cache \
g++ \
autoconf \
automake \
patch \
libtool \
musl-dev \
pcre-dev \
@ -43,6 +44,11 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2
@ -155,6 +161,9 @@ COPY setup/app ${APP_DIR}
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1

View File

@ -0,0 +1,18 @@
--- ckan/ckanext/datapusher/logic/action.py 2020-09-19 21:25:06.525350202 +0200
+++ action_fix_callback_url.py 2020-09-19 21:27:43.596501963 +0200
@@ -61,13 +61,13 @@
datapusher_url = config.get('ckan.datapusher.url')
- site_url = h.url_for('/', qualified=True)
-
callback_url_base = config.get('ckan.datapusher.callback_url_base')
if callback_url_base:
+ site_url = callback_url_base
callback_url = urlparse.urljoin(
callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook')
else:
+ site_url = h.url_for('/', qualified=True)
callback_url = h.url_for(
'/api/3/action/datapusher_hook', qualified=True)

View File

@ -0,0 +1,5 @@
#!/bin/bash
shopt -s nullglob
for patch in patches/*.patch; do
/usr/bin/patch -p0 -i $patch
done

View File

@ -18,6 +18,19 @@ UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --e
# Run the prerun script to init CKAN and create the default admin user
python prerun.py
# Run any after prerun/init scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
then
for f in ${APP_DIR}/docker-afterinit.d/*; do
case "$f" in
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then

View File

@ -26,6 +26,7 @@ RUN apk add --no-cache \
autoconf \
automake \
libtool \
patch \
musl-dev \
pcre-dev \
pcre \
@ -49,6 +50,11 @@ RUN pip install setuptools==44.1.0
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==20.6.2
@ -170,6 +176,9 @@ COPY setup/app ${APP_DIR}
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1

View File

@ -0,0 +1,5 @@
#!/bin/bash
shopt -s nullglob
for patch in patches/*.patch; do
/usr/bin/patch -p0 -i $patch
done

View File

@ -18,6 +18,19 @@ UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid ckan --gid ckan --http :5000 --master
# Run the prerun script to init CKAN and create the default admin user
python prerun.py
# Run any after prerun/init scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
then
for f in ${APP_DIR}/docker-afterinit.d/*; do
case "$f" in
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then

View File

@ -5,7 +5,7 @@ FROM alpine:3.12 as build
# Set datapusher version to build
ENV GIT_URL https://github.com/keitaroinc/datapusher.git
ENV GIT_BRANCH parametrize-job-config
ENV GIT_BRANCH master
ENV REQUIREMENTS_URL https://raw.githubusercontent.com/keitaroinc/datapusher/${GIT_BRANCH}/requirements.txt
# Set src dirs

View File

@ -31,3 +31,7 @@ MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '102400
CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384'))
CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250'))
DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30'))
# Rewrite resource URL's when ckan callback url base is used
REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', False)
REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/')