From ad9918efad2440ec32a8685fdbd0af3d2e497fb5 Mon Sep 17 00:00:00 2001 From: Brett Date: Sun, 2 Apr 2023 17:34:49 +1200 Subject: [PATCH 1/6] Update .env and .env.example --- .env | 2 +- .env.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 377177a..21d25d4 100644 --- a/.env +++ b/.env @@ -59,7 +59,7 @@ CKAN_REDIS_URL=redis://redis:6379/1 TEST_CKAN_REDIS_URL=redis://redis:6379/1 # Datapusher -DATAPUSHER_VERSION=0.0.19 +DATAPUSHER_VERSION=0.0.20 CKAN_DATAPUSHER_URL=http://datapusher:8800 CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 DATAPUSHER_REWRITE_RESOURCES=True diff --git a/.env.example b/.env.example index f31d1ee..cc9220a 100644 --- a/.env.example +++ b/.env.example @@ -59,7 +59,7 @@ CKAN_REDIS_URL=redis://redis:6379/1 TEST_CKAN_REDIS_URL=redis://redis:6379/1 # Datapusher -DATAPUSHER_VERSION=0.0.19 +DATAPUSHER_VERSION=0.0.20 CKAN_DATAPUSHER_URL=http://datapusher:8800 CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 DATAPUSHER_REWRITE_RESOURCES=True From 0a876246191e4ca651ae612f3ce5691bca6291cf Mon Sep 17 00:00:00 2001 From: Brett Jones <54408245+kowh-ai@users.noreply.github.com> Date: Wed, 5 Apr 2023 13:43:41 +1200 Subject: [PATCH 2/6] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 450aada..dd01310 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ The Docker image config files used to build your CKAN project are located in the * CKAN is started running this: `/usr/bin/ckan -c /srv/app/ckan.ini run -H 0.0.0.0`. * Make sure to add the local plugins to the `CKAN__PLUGINS` env var in the `.env` file. +* Any custom changes to the scripts run during container start up can be made to scripts in the `setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, and the `start_ckan.sh.override` file. Then you would need to add the following line to the Dockerfile ie: `COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh` ## 6. Extending the base images From 1f0d66086dabb5c8299265d441ac0c5e1d0d5c90 Mon Sep 17 00:00:00 2001 From: Brett Jones <54408245+kowh-ai@users.noreply.github.com> Date: Wed, 5 Apr 2023 13:51:59 +1200 Subject: [PATCH 3/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dd01310..295ee86 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ The Docker image config files used to build your CKAN project are located in the * CKAN is started running this: `/usr/bin/ckan -c /srv/app/ckan.ini run -H 0.0.0.0`. * Make sure to add the local plugins to the `CKAN__PLUGINS` env var in the `.env` file. -* Any custom changes to the scripts run during container start up can be made to scripts in the `setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, and the `start_ckan.sh.override` file. Then you would need to add the following line to the Dockerfile ie: `COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh` +* Any custom changes to the scripts run during container start up can be made to scripts in the `setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, and the `start_ckan.sh.override` file. Then you would need to add the following line to the Dockerfile ie: `COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh`. The `start_ckan.sh` file in the locally built image would override the `start_ckan.sh` file included in the base image ## 6. Extending the base images From 7c010b370c494082e57274b361140e4289ad74ff Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 5 Apr 2023 13:55:59 +1200 Subject: [PATCH 4/6] Update setup/ directory files --- ckan/setup/{prerun.py => prerun.py.override} | 1 + .../{start_ckan.sh => start_ckan.sh.override} | 20 +++- ckan/setup/start_ckan_development.sh.override | 99 +++++++++++++++++++ ckan/setup/supervisord.conf | 23 ----- ckan/setup/uwsgi.conf | 2 - 5 files changed, 119 insertions(+), 26 deletions(-) rename ckan/setup/{prerun.py => prerun.py.override} (99%) rename ckan/setup/{start_ckan.sh => start_ckan.sh.override} (51%) create mode 100755 ckan/setup/start_ckan_development.sh.override delete mode 100644 ckan/setup/supervisord.conf delete mode 100644 ckan/setup/uwsgi.conf diff --git a/ckan/setup/prerun.py b/ckan/setup/prerun.py.override similarity index 99% rename from ckan/setup/prerun.py rename to ckan/setup/prerun.py.override index b148d78..3d68696 100644 --- a/ckan/setup/prerun.py +++ b/ckan/setup/prerun.py.override @@ -209,3 +209,4 @@ if __name__ == "__main__": init_datastore_db() check_solr_connection() create_sysadmin() + \ No newline at end of file diff --git a/ckan/setup/start_ckan.sh b/ckan/setup/start_ckan.sh.override similarity index 51% rename from ckan/setup/start_ckan.sh rename to ckan/setup/start_ckan.sh.override index e1fb8e1..0c8409c 100755 --- a/ckan/setup/start_ckan.sh +++ b/ckan/setup/start_ckan.sh.override @@ -1,8 +1,26 @@ -#!/bin/bash +#!/bin/sh + +# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) +ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx + +# Set up the Secret key used by Beaker and Flask +# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var +if grep -E "beaker.session.secret ?= ?$" ckan.ini +then + echo "Setting beaker.session.secret in ini file" + ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" + ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi # Run the prerun script to init CKAN and create the default admin user sudo -u ckan -EH python3 prerun.py +echo "Set up ckan.datapusher.api_token in the CKAN config file" +ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" + # Run any startup scripts provided by images extending this one if [[ -d "/docker-entrypoint.d" ]] then diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override new file mode 100755 index 0000000..93ce814 --- /dev/null +++ b/ckan/setup/start_ckan_development.sh.override @@ -0,0 +1,99 @@ +#!/bin/sh + +# Install any local extensions in the src_extensions volume +echo "Looking for local extensions to install..." +echo "Extension dir contents:" +ls -la $SRC_EXTENSIONS_DIR +for i in $SRC_EXTENSIONS_DIR/* +do + if [ -d $i ]; + then + + if [ -f $i/pip-requirements.txt ]; + then + pip install -r $i/pip-requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/requirements.txt ]; + then + pip install -r $i/requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/dev-requirements.txt ]; + then + pip install -r $i/dev-requirements.txt + echo "Found dev-requirements file in $i" + fi + if [ -f $i/setup.py ]; + then + cd $i + python3 $i/setup.py develop + echo "Found setup.py file in $i" + cd $APP_DIR + fi + + # Point `use` in test.ini to location of `test-core.ini` + if [ -f $i/test.ini ]; + then + echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" + ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + fi + fi +done + +# Set debug to true +echo "Enabling debug mode" +ckan config-tool $CKAN_INI -s DEFAULT "debug = true" + +# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) +ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx + +# Set up the Secret key used by Beaker and Flask +# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var +if grep -E "beaker.session.secret ?= ?$" ckan.ini +then + echo "Setting beaker.session.secret in ini file" + ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" + ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi + +# Update the plugins setting in the ini file with the values defined in the env var +echo "Loading the following plugins: $CKAN__PLUGINS" +ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" + +# Update test-core.ini DB, SOLR & Redis settings +echo "Loading test settings into test-core.ini" +ckan config-tool $SRC_DIR/ckan/test-core.ini \ + "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ + "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ + "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ + "solr_url = $TEST_CKAN_SOLR_URL" \ + "ckan.redis.url = $TEST_CKAN_REDIS_URL" + +# Run the prerun script to init CKAN and create the default admin user +sudo -u ckan -EH python3 prerun.py + +echo "Set up ckan.datapusher.api_token in the CKAN config file" +ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Start supervisord +supervisord --configuration /etc/supervisord.conf & + +# Start the development server with automatic reload +sudo -u ckan -EH ckan -c $CKAN_INI run -H 0.0.0.0 \ No newline at end of file diff --git a/ckan/setup/supervisord.conf b/ckan/setup/supervisord.conf deleted file mode 100644 index a3f6671..0000000 --- a/ckan/setup/supervisord.conf +++ /dev/null @@ -1,23 +0,0 @@ -[unix_http_server] -file = /tmp/supervisor.sock -chmod = 0777 -chown = nobody:nogroup - -[supervisord] -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = info -pidfile = /tmp/supervisord.pid -nodaemon = true -umask = 022 -identifier = supervisor - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[include] -files = /etc/supervisord.d/*.conf diff --git a/ckan/setup/uwsgi.conf b/ckan/setup/uwsgi.conf deleted file mode 100644 index 6321d6d..0000000 --- a/ckan/setup/uwsgi.conf +++ /dev/null @@ -1,2 +0,0 @@ -[uwsgi] -route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd From d3152d3f7c8f6d36aafe367dc011234eb358440c Mon Sep 17 00:00:00 2001 From: Brett Jones <54408245+kowh-ai@users.noreply.github.com> Date: Mon, 15 May 2023 03:23:32 +1200 Subject: [PATCH 5/6] Update README.md Added CKAN_SITE_URL --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 295ee86..a51544c 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ * [Datastore and Datapusher](#Datastore-and-datapusher) * [NGINX](#nginx) * [The ckanext-envvars extension](#envvars) +* [The CKAN_SITE_URL parameter](#CKAN_SITE_URL) ## 1. Overview @@ -228,3 +229,7 @@ For example: These parameters can be added to the `.env` file For more information please see [ckanext-envvars](https://github.com/okfn/ckanext-envvars) + +## 12. CKAN_SITE_URL + +For convenience the CKAN_SITE_URL parameter should be set in the .env file. For development it can be set to http://localhost:5000 and non-development set to https://localhost:8443 From 5a9c23eb7fa57f84d048189e2996614af511b6d6 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 25 May 2023 14:06:50 +0200 Subject: [PATCH 6/6] Updates to Dockerfiles and doco --- README.md | 6 ++++++ ckan/Dockerfile | 4 ++-- ckan/Dockerfile.dev | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a51544c..6b1931c 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ * [NGINX](#nginx) * [The ckanext-envvars extension](#envvars) * [The CKAN_SITE_URL parameter](#CKAN_SITE_URL) +* [Changing the base image](#Changing-the-base-image) ## 1. Overview @@ -233,3 +234,8 @@ For more information please see [ckanext-envvars](https://github.com/okfn/ckanex ## 12. CKAN_SITE_URL For convenience the CKAN_SITE_URL parameter should be set in the .env file. For development it can be set to http://localhost:5000 and non-development set to https://localhost:8443 + +## 13. Changing the base image + +The base image used in the CKAN Dockerfile and Dockerfile.dev can be changed so a different DockerHub image is used eg: ckan/ckan-base:2.9.9 +could be used instead of ckan/ckan-base:2.10.1 diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 89b67a8..75001e9 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,5 +1,5 @@ -FROM ckan/ckan-base:ckan-2.10.0 -#FROM ckan/ckan-base:dev-v2.10 +FROM ckan/ckan-base:2.10.1 +#FROM ckan/ckan-base:2.9.9 # Set up environment variables ENV APP_DIR=/srv/app diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index cd97021..8488f0b 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,4 +1,5 @@ -FROM ckan/ckan-base:ckan-2.10.0-dev +FROM ckan/ckan-base:2.10.1-dev +#FROM ckan/ckan-base:2.9.9-dev # Set up environment variables