From 986068ba68166ec97134af227035703ed45c16de Mon Sep 17 00:00:00 2001 From: Brett <54408245+kowh-ai@users.noreply.github.com> Date: Fri, 29 Oct 2021 17:05:48 +0200 Subject: [PATCH] Revert "New changes (29 Oct 2021)" --- .env | 58 ++++-- .gitkeep | 0 Dockerfile | 35 ---- Dockerfile.dev | 37 ---- README | 12 -- ckan-base/2.7/Dockerfile | 101 --------- ckan-base/2.7/setup/prerun.py | 197 ------------------ ckan-base/2.7/setup/start_ckan.sh | 47 ----- ckan-base/2.7/setup/supervisor.worker.conf | 13 -- ckan-base/2.7/setup/supervisord.conf | 23 -- ckan-base/2.7/setup/uwsgi.conf | 2 - ckan-base/2.8/Dockerfile | 99 --------- ckan-base/2.8/setup/prerun.py | 197 ------------------ ckan-base/2.8/setup/start_ckan.sh | 47 ----- ckan-base/2.8/setup/supervisor.worker.conf | 13 -- ckan-base/2.8/setup/supervisord.conf | 23 -- ckan-base/2.8/setup/uwsgi.conf | 2 - ckan-base/2.9/Dockerfile | 109 ---------- ckan-base/2.9/setup/ckan-uwsgi.ini | 15 -- ckan-base/2.9/setup/prerun.py | 194 ----------------- ckan-base/2.9/setup/start_ckan.sh | 46 ---- ckan-base/2.9/setup/supervisor.worker.conf | 12 -- ckan-base/2.9/setup/supervisord.conf | 23 -- ckan-base/2.9/setup/wsgi.py | 9 - ckan-dev/2.7/Dockerfile | 20 -- ckan-dev/2.8/Dockerfile | 20 -- ckan-dev/2.9/Dockerfile | 29 --- ckan-dev/setup/start_ckan_development.sh | 82 -------- docker-compose.yml | 48 ++--- .../10_create_datastore.sql | 4 - .../20_setup_test_databases.sql | 2 - index.html | 22 -- notes.txt | 69 ------ patches/00_patch_sql_url.patch | 11 - .../01_patch_resource_replace_upload.patch | 11 - .../02_patch_postgres_username_split.patch | 11 - scripts/apply_ckan_patches.sh | 5 - setup/ckan-uwsgi.ini | 15 -- setup/datapusher-uwsgi.ini | 14 -- setup/datapusher_settings.py | 37 ---- setup/nginx.conf | 34 --- setup/prerun.py | 194 ----------------- setup/sites-available/ckan.conf | 18 -- setup/start_ckan.sh | 46 ---- setup/start_datapusher.sh | 5 - setup/supervisor.uwsgi.conf | 12 -- setup/supervisor.worker.conf | 12 -- setup/supervisord.conf | 23 -- setup/wsgi.py | 9 - solr/Dockerfile | 24 +-- solr/{solrconfig-2.9.3.xml => solrconfig.xml} | 2 +- todo.txt | 12 -- 52 files changed, 65 insertions(+), 2040 deletions(-) delete mode 100644 .gitkeep delete mode 100644 Dockerfile delete mode 100644 Dockerfile.dev delete mode 100755 README delete mode 100755 ckan-base/2.7/Dockerfile delete mode 100755 ckan-base/2.7/setup/prerun.py delete mode 100755 ckan-base/2.7/setup/start_ckan.sh delete mode 100755 ckan-base/2.7/setup/supervisor.worker.conf delete mode 100755 ckan-base/2.7/setup/supervisord.conf delete mode 100755 ckan-base/2.7/setup/uwsgi.conf delete mode 100755 ckan-base/2.8/Dockerfile delete mode 100755 ckan-base/2.8/setup/prerun.py delete mode 100755 ckan-base/2.8/setup/start_ckan.sh delete mode 100755 ckan-base/2.8/setup/supervisor.worker.conf delete mode 100755 ckan-base/2.8/setup/supervisord.conf delete mode 100755 ckan-base/2.8/setup/uwsgi.conf delete mode 100755 ckan-base/2.9/Dockerfile delete mode 100644 ckan-base/2.9/setup/ckan-uwsgi.ini delete mode 100755 ckan-base/2.9/setup/prerun.py delete mode 100755 ckan-base/2.9/setup/start_ckan.sh delete mode 100644 ckan-base/2.9/setup/supervisor.worker.conf delete mode 100644 ckan-base/2.9/setup/supervisord.conf delete mode 100644 ckan-base/2.9/setup/wsgi.py delete mode 100755 ckan-dev/2.7/Dockerfile delete mode 100755 ckan-dev/2.8/Dockerfile delete mode 100755 ckan-dev/2.9/Dockerfile delete mode 100755 ckan-dev/setup/start_ckan_development.sh delete mode 100755 docker-entrypoint-initdb.d/10_create_datastore.sql delete mode 100755 docker-entrypoint-initdb.d/20_setup_test_databases.sql delete mode 100644 index.html delete mode 100644 notes.txt delete mode 100644 patches/00_patch_sql_url.patch delete mode 100644 patches/01_patch_resource_replace_upload.patch delete mode 100644 patches/02_patch_postgres_username_split.patch delete mode 100755 scripts/apply_ckan_patches.sh delete mode 100644 setup/ckan-uwsgi.ini delete mode 100644 setup/datapusher-uwsgi.ini delete mode 100644 setup/datapusher_settings.py delete mode 100644 setup/nginx.conf delete mode 100755 setup/prerun.py delete mode 100644 setup/sites-available/ckan.conf delete mode 100755 setup/start_ckan.sh delete mode 100755 setup/start_datapusher.sh delete mode 100644 setup/supervisor.uwsgi.conf delete mode 100644 setup/supervisor.worker.conf delete mode 100644 setup/supervisord.conf delete mode 100644 setup/wsgi.py rename solr/{solrconfig-2.9.3.xml => solrconfig.xml} (99%) mode change 100644 => 100755 delete mode 100644 todo.txt diff --git a/.env b/.env index 211d393..a3d9300 100644 --- a/.env +++ b/.env @@ -1,10 +1,4 @@ -# CKAN -CKAN_VERSION=2.9.3 -CKAN_SITE_URL=http://localhost:5000 -CKAN_SITE_ID=default -CKAN_PORT=5000 - -# Database +# CKAN databases POSTGRES_USER=ckan POSTGRES_PASSWORD=ckan DATASTORE_READONLY_USER=datastore_ro @@ -14,21 +8,39 @@ CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore -# solr -SOLR_VERSION=6.6.6 -CKAN_SOLR_URL=http://solr:8983/solr/ckan +# Test database connections +TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test +TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test -# Datapusher -DATAPUSHER_VERSION=0.0.17 -CKAN_DATAPUSHER_URL=http://datapusher:8800 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 -DATAPUSHER_REWRITE_RESOURCES=True -DATAPUSHER_REWRITE_URL=http://ckan:5000 - -# Redis -REDIS_VERSION=6.0.7 -CKAN_REDIS_URL=redis://redis:6379/1 - -CKAN_SYSADMIN_NAME=admin +# CKAN core +CKAN_SITE_ID=default +CKAN_SITE_URL=http://ckan:5000 +CKAN_PORT=5000 +CKAN_SYSADMIN_NAME=ckan_admin CKAN_SYSADMIN_PASSWORD=test1234 -CKAN_SYSADMIN_EMAIL=admin@ckan.org +CKAN_SYSADMIN_EMAIL=your_email@example.com +CKAN_STORAGE_PATH=/var/lib/ckan +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost +TZ=UTC + +# Other services +CKAN_SOLR_URL=http://solr:8983/solr/ckan +CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_DATAPUSHER_URL=http://datapusher:8801 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 + +# test connections +TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan +TEST_CKAN_REDIS_URL=redis://redis:6379/1 + +# Extensions +CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__PORT=6379 +CKAN__HARVEST__MQ__REDIS_DB=1 \ No newline at end of file diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a29437d..0000000 --- a/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM solr:6.6.6 - -# Enviroment variables -ENV SOLR_CORE ckan - -# Build Arguments -ARG CKAN_VERSION -ARG SOLR_VERSION - -# root user for initial config -USER root - -# Create directories -RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \ - mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data && \ - mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index - -# Add files -COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml -ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ -/opt/solr/server/solr/${SOLR_CORE}/conf/ - -# Create core.properties -RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties - -# Giving ownership to Solr -RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE} - -# non-root user for runtime -USER ${SOLR_USER}:${SOLR_USER} \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 6086f6e..0000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,37 +0,0 @@ -FROM ckan/ckan-dev:testing-only.2.9 - -LABEL maintainer="brett@kowh.ai" - -# Set up environment variables -ENV APP_DIR=/srv/app -ENV TZ=UTC -RUN echo ${TZ} > /etc/timezone - -# Make sure both files are not exactly the same -RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \ - cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\ - fi ; - -# Install any extensions needed by your CKAN instance -# (Make sure to add the plugins to CKAN__PLUGINS in the .env file) -# For instance: -#RUN pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages && \ -# pip install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \ -# pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt - -# Clone the extension(s) your are writing for your own project in the `src` folder -# to get them mounted in this image at runtime - -# Apply any patches needed to CKAN core or any of the built extensions (not the -# runtime mounted ones) -# See https://github.com/okfn/docker-ckan#applying-patches - -COPY patches ${APP_DIR}/patches - -RUN for d in ${APP_DIR}/patches/*; do \ - if [ -d $d ]; then \ - for f in `ls $d/*.patch | sort -g`; do \ - cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \ - done ; \ - fi ; \ - done diff --git a/README b/README deleted file mode 100755 index 9c85779..0000000 --- a/README +++ /dev/null @@ -1,12 +0,0 @@ -Build ckan-base image (2.9) -cd /path/to/ckan-docker/ckan/ckan-base -docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile . - -Login to Dockerhub -docker login --username=kowhai -(will then ask for a password) - -Push image to (ckan) Dockerhub -docker push ckan/ckan-base:testing-only.2.9 - -Do the same with 2.8 and 2.7 diff --git a/ckan-base/2.7/Dockerfile b/ckan-base/2.7/Dockerfile deleted file mode 100755 index 4eb4332..0000000 --- a/ckan-base/2.7/Dockerfile +++ /dev/null @@ -1,101 +0,0 @@ -FROM alpine:3.13 - -# Internals, you probably don't need to change these -ENV APP_DIR=/srv/app -ENV SRC_DIR=/srv/app/src -ENV CKAN_INI=${APP_DIR}/production.ini -ENV PIP_SRC=${SRC_DIR} -ENV CKAN_STORAGE_PATH=/var/lib/ckan -ENV GIT_URL=https://github.com/ckan/ckan.git -# CKAN version to build -ENV GIT_BRANCH=ckan-2.7.11 -# Customize these on the .env file if needed -ENV CKAN_SITE_URL=http://localhost:5000 -ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars - -WORKDIR ${APP_DIR} - -# Install necessary packages to run CKAN -RUN apk add --no-cache tzdata \ - git \ - gettext \ - postgresql-client \ - python \ - apache2-utils \ - libxml2 \ - libxslt \ - musl-dev \ - uwsgi \ - uwsgi-http \ - uwsgi-corerouter \ - uwsgi-python \ - py2-gevent \ - uwsgi-gevent \ - libmagic \ - curl \ - sudo && \ - # Packages to build CKAN requirements and plugins - apk add --no-cache --virtual .build-deps \ - postgresql-dev \ - gcc \ - make \ - g++ \ - autoconf \ - automake \ - libtool \ - python-dev \ - libxml2-dev \ - libxslt-dev \ - linux-headers - -# Create SRC_DIR -RUN mkdir -p ${SRC_DIR} - -# Install pip -RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ - python ${SRC_DIR}/get-pip.py 'pip==20.3.3' - -# Install pip, supervisord and uwsgi -RUN pip install supervisor && \ - mkdir /etc/supervisord.d && \ - #pip wheel --wheel-dir=/wheels uwsgi gevent && \ - rm -rf ${SRC_DIR}/get-pip.py - -COPY 2.7/setup/supervisord.conf /etc - -# Install CKAN -RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ - cd ${SRC_DIR}/ckan && \ - cp who.ini ${APP_DIR} && \ - # Workaround to solve https://github.com/psycopg/psycopg2/issues/594 in Alpine 3.7 - sed -i -e "s/psycopg2==2.4.5/psycopg2==2.7.3.2/g" requirements.txt && \ - pip install --no-binary :all: -r requirements.txt && \ - # Install CKAN envvars to support loading config from environment variables - pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \ - # Create and update CKAN config - paster --plugin=ckan make-config ckan ${CKAN_INI} && \ - paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \ - paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}" - -# Create a local user and group to run the app -RUN addgroup -g 92 -S ckan && \ - adduser -u 92 -h /srv/app -H -D -S -G ckan ckan - -# Create local storage folder -RUN mkdir -p $CKAN_STORAGE_PATH && \ - chown -R ckan:ckan $CKAN_STORAGE_PATH - -COPY 2.7/setup ${APP_DIR} -COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf -COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf - -# Create entrypoint directory for children image scripts -ONBUILD RUN mkdir /docker-entrypoint.d - -RUN chown ckan -R /srv/app - -EXPOSE 5000 - -HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 - -CMD ["/srv/app/start_ckan.sh"] diff --git a/ckan-base/2.7/setup/prerun.py b/ckan-base/2.7/setup/prerun.py deleted file mode 100755 index 7ebacfa..0000000 --- a/ckan-base/2.7/setup/prerun.py +++ /dev/null @@ -1,197 +0,0 @@ -import os -import sys -import subprocess -import psycopg2 -import urllib2 -import time -import re - -ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini') - -RETRY = 5 - -def update_plugins(): - - plugins = os.environ.get('CKAN__PLUGINS', '') - print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) - print(plugins) - cmd = ['paster', '--plugin=ckan', 'config-tool', - ckan_ini, 'ckan.plugins = {}'.format(plugins)] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - print '[prerun] Plugins set.' - - -def check_main_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') - if not conn_str: - print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db' - return check_db_connection(conn_str, retry) - - -def check_datastore_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db' - return check_db_connection(conn_str, retry) - - -def check_db_connection(conn_str, retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print '[prerun] Giving up after 5 tries...' - sys.exit(1) - - try: - connection = psycopg2.connect(conn_str) - - except psycopg2.Error as e: - print str(e) - print '[prerun] Unable to connect to the database, waiting...' - time.sleep(10) - check_db_connection(conn_str, retry=retry - 1) - else: - connection.close() - - -def check_solr_connection(retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print '[prerun] Giving up after 5 tries...' - sys.exit(1) - - url = os.environ.get('CKAN_SOLR_URL', '') - search_url = '{url}/select/?q=*&wt=json'.format(url=url) - - try: - connection = urllib2.urlopen(search_url) - except urllib2.URLError as e: - print str(e) - print '[prerun] Unable to connect to solr, waiting...' - time.sleep(10) - check_solr_connection(retry=retry - 1) - else: - eval(connection.read()) - - -def init_db(): - - db_command = ['paster', '--plugin=ckan', 'db', - 'init', '-c', ckan_ini] - print '[prerun] Initializing or upgrading db - start' - try: - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print '[prerun] Initializing or upgrading db - end' - except subprocess.CalledProcessError, e: - if 'OperationalError' in e.output: - print e.output - print '[prerun] Database not ready, waiting a bit before exit...' - time.sleep(5) - sys.exit(1) - else: - print e.output - raise e - - -def init_datastore_db(): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print '[prerun] Skipping datastore initialization' - return - - datastore_perms_command = ['paster', '--plugin=ckan', 'datastore', - 'set-permissions', '-c', ckan_ini] - - connection = psycopg2.connect(conn_str) - cursor = connection.cursor() - - print '[prerun] Initializing datastore db - start' - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, - stdout=subprocess.PIPE) - - perms_sql = datastore_perms.stdout.read() - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print notice - - connection.commit() - - print '[prerun] Initializing datastore db - end' - print datastore_perms.stdout.read() - except psycopg2.Error as e: - print '[prerun] Could not initialize datastore' - print str(e) - - except subprocess.CalledProcessError, e: - if 'OperationalError' in e.output: - print e.output - print '[prerun] Database not ready, waiting a bit before exit...' - time.sleep(5) - sys.exit(1) - else: - print e.output - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - name = os.environ.get('CKAN_SYSADMIN_NAME') - password = os.environ.get('CKAN_SYSADMIN_PASSWORD') - email = os.environ.get('CKAN_SYSADMIN_EMAIL') - - if name and password and email: - - # Check if user exists - command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini] - - out = subprocess.check_output(command) - if 'User:None' not in re.sub(r'\s', '', out): - print '[prerun] Sysadmin user exists, skipping creation' - return - - # Create user - command = ['paster', '--plugin=ckan', 'user', 'add', - name, - 'password=' + password, - 'email=' + email, - '-c', ckan_ini] - - subprocess.call(command) - print '[prerun] Created user {0}'.format(name) - - # Make it sysadmin - command = ['paster', '--plugin=ckan', 'sysadmin', 'add', - name, - '-c', ckan_ini] - - subprocess.call(command) - print '[prerun] Made user {0} a sysadmin'.format(name) - - -if __name__ == '__main__': - - maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' - - if maintenance: - print '[prerun] Maintenance mode, skipping setup...' - else: - check_main_db_connection() - init_db() - update_plugins() - check_datastore_db_connection() - init_datastore_db() - check_solr_connection() - create_sysadmin() \ No newline at end of file diff --git a/ckan-base/2.7/setup/start_ckan.sh b/ckan-base/2.7/setup/start_ckan.sh deleted file mode 100755 index d1c6726..0000000 --- a/ckan-base/2.7/setup/start_ckan.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python prerun.py - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - -# Set the common uwsgi options -UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L" - -# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully -if [ $? -eq 0 ] -then - if [ "$PASSWORD_PROTECT" = true ] - then - if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] - then - # Generate htpasswd file for basicauth - htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi with basicauth - sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS - else - echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." - exit 1 - fi - else - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi - sudo -u ckan -EH uwsgi $UWSGI_OPTS - fi -else - echo "[prerun] failed...not starting CKAN." -fi diff --git a/ckan-base/2.7/setup/supervisor.worker.conf b/ckan-base/2.7/setup/supervisor.worker.conf deleted file mode 100755 index 0ca43ba..0000000 --- a/ckan-base/2.7/setup/supervisor.worker.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:ckan-worker] -command=paster --plugin=ckan jobs worker -c /srv/app/production.ini -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" - diff --git a/ckan-base/2.7/setup/supervisord.conf b/ckan-base/2.7/setup/supervisord.conf deleted file mode 100755 index a3f6671..0000000 --- a/ckan-base/2.7/setup/supervisord.conf +++ /dev/null @@ -1,23 +0,0 @@ -[unix_http_server] -file = /tmp/supervisor.sock -chmod = 0777 -chown = nobody:nogroup - -[supervisord] -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = info -pidfile = /tmp/supervisord.pid -nodaemon = true -umask = 022 -identifier = supervisor - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[include] -files = /etc/supervisord.d/*.conf diff --git a/ckan-base/2.7/setup/uwsgi.conf b/ckan-base/2.7/setup/uwsgi.conf deleted file mode 100755 index 6321d6d..0000000 --- a/ckan-base/2.7/setup/uwsgi.conf +++ /dev/null @@ -1,2 +0,0 @@ -[uwsgi] -route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd diff --git a/ckan-base/2.8/Dockerfile b/ckan-base/2.8/Dockerfile deleted file mode 100755 index 59904a3..0000000 --- a/ckan-base/2.8/Dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -FROM alpine:3.13 - -# Internals, you probably don't need to change these -ENV APP_DIR=/srv/app -ENV SRC_DIR=/srv/app/src -ENV CKAN_INI=${APP_DIR}/production.ini -ENV PIP_SRC=${SRC_DIR} -ENV CKAN_STORAGE_PATH=/var/lib/ckan -ENV GIT_URL=https://github.com/ckan/ckan.git -# CKAN version to build -ENV GIT_BRANCH=ckan-2.8.8 -# Customize these on the .env file if needed -ENV CKAN_SITE_URL=http://localhost:5000 -ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars - -WORKDIR ${APP_DIR} - -# Install necessary packages to run CKAN -RUN apk add --no-cache tzdata \ - git \ - gettext \ - postgresql-client \ - python \ - apache2-utils \ - libxml2 \ - libxslt \ - musl-dev \ - uwsgi \ - uwsgi-http \ - uwsgi-corerouter \ - uwsgi-python \ - py2-gevent \ - uwsgi-gevent \ - libmagic \ - curl \ - sudo && \ - # Packages to build CKAN requirements and plugins - apk add --no-cache --virtual .build-deps \ - postgresql-dev \ - gcc \ - make \ - g++ \ - autoconf \ - automake \ - libtool \ - python-dev \ - libxml2-dev \ - libxslt-dev \ - linux-headers - -# Create SRC_DIR -RUN mkdir -p ${SRC_DIR} - -# Install pip -RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ - python ${SRC_DIR}/get-pip.py 'pip==20.3.3' - -# Install supervisord and uwsgi -RUN pip install supervisor && \ - mkdir /etc/supervisord.d && \ - #pip wheel --wheel-dir=/wheels uwsgi gevent && \ - rm -rf ${SRC_DIR}/get-pip.py - -COPY 2.8/setup/supervisord.conf /etc - -# Install CKAN -RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ - cd ${SRC_DIR}/ckan && \ - cp who.ini ${APP_DIR} && \ - pip install --no-binary :all: -r requirements.txt && \ - # Install CKAN envvars to support loading config from environment variables - pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \ - # Create and update CKAN config - paster --plugin=ckan make-config ckan ${CKAN_INI} && \ - paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \ - paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}" - -# Create a local user and group to run the app -RUN addgroup -g 92 -S ckan && \ - adduser -u 92 -h /srv/app -H -D -S -G ckan ckan - -# Create local storage folder -RUN mkdir -p $CKAN_STORAGE_PATH && \ - chown -R ckan:ckan $CKAN_STORAGE_PATH - -COPY 2.8/setup ${APP_DIR} -COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf -COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf - -# Create entrypoint directory for children image scripts -ONBUILD RUN mkdir /docker-entrypoint.d - -RUN chown ckan -R /srv/app - -EXPOSE 5000 - -HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 - -CMD ["/srv/app/start_ckan.sh"] diff --git a/ckan-base/2.8/setup/prerun.py b/ckan-base/2.8/setup/prerun.py deleted file mode 100755 index 7ebacfa..0000000 --- a/ckan-base/2.8/setup/prerun.py +++ /dev/null @@ -1,197 +0,0 @@ -import os -import sys -import subprocess -import psycopg2 -import urllib2 -import time -import re - -ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini') - -RETRY = 5 - -def update_plugins(): - - plugins = os.environ.get('CKAN__PLUGINS', '') - print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) - print(plugins) - cmd = ['paster', '--plugin=ckan', 'config-tool', - ckan_ini, 'ckan.plugins = {}'.format(plugins)] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - print '[prerun] Plugins set.' - - -def check_main_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') - if not conn_str: - print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db' - return check_db_connection(conn_str, retry) - - -def check_datastore_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db' - return check_db_connection(conn_str, retry) - - -def check_db_connection(conn_str, retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print '[prerun] Giving up after 5 tries...' - sys.exit(1) - - try: - connection = psycopg2.connect(conn_str) - - except psycopg2.Error as e: - print str(e) - print '[prerun] Unable to connect to the database, waiting...' - time.sleep(10) - check_db_connection(conn_str, retry=retry - 1) - else: - connection.close() - - -def check_solr_connection(retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print '[prerun] Giving up after 5 tries...' - sys.exit(1) - - url = os.environ.get('CKAN_SOLR_URL', '') - search_url = '{url}/select/?q=*&wt=json'.format(url=url) - - try: - connection = urllib2.urlopen(search_url) - except urllib2.URLError as e: - print str(e) - print '[prerun] Unable to connect to solr, waiting...' - time.sleep(10) - check_solr_connection(retry=retry - 1) - else: - eval(connection.read()) - - -def init_db(): - - db_command = ['paster', '--plugin=ckan', 'db', - 'init', '-c', ckan_ini] - print '[prerun] Initializing or upgrading db - start' - try: - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print '[prerun] Initializing or upgrading db - end' - except subprocess.CalledProcessError, e: - if 'OperationalError' in e.output: - print e.output - print '[prerun] Database not ready, waiting a bit before exit...' - time.sleep(5) - sys.exit(1) - else: - print e.output - raise e - - -def init_datastore_db(): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print '[prerun] Skipping datastore initialization' - return - - datastore_perms_command = ['paster', '--plugin=ckan', 'datastore', - 'set-permissions', '-c', ckan_ini] - - connection = psycopg2.connect(conn_str) - cursor = connection.cursor() - - print '[prerun] Initializing datastore db - start' - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, - stdout=subprocess.PIPE) - - perms_sql = datastore_perms.stdout.read() - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print notice - - connection.commit() - - print '[prerun] Initializing datastore db - end' - print datastore_perms.stdout.read() - except psycopg2.Error as e: - print '[prerun] Could not initialize datastore' - print str(e) - - except subprocess.CalledProcessError, e: - if 'OperationalError' in e.output: - print e.output - print '[prerun] Database not ready, waiting a bit before exit...' - time.sleep(5) - sys.exit(1) - else: - print e.output - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - name = os.environ.get('CKAN_SYSADMIN_NAME') - password = os.environ.get('CKAN_SYSADMIN_PASSWORD') - email = os.environ.get('CKAN_SYSADMIN_EMAIL') - - if name and password and email: - - # Check if user exists - command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini] - - out = subprocess.check_output(command) - if 'User:None' not in re.sub(r'\s', '', out): - print '[prerun] Sysadmin user exists, skipping creation' - return - - # Create user - command = ['paster', '--plugin=ckan', 'user', 'add', - name, - 'password=' + password, - 'email=' + email, - '-c', ckan_ini] - - subprocess.call(command) - print '[prerun] Created user {0}'.format(name) - - # Make it sysadmin - command = ['paster', '--plugin=ckan', 'sysadmin', 'add', - name, - '-c', ckan_ini] - - subprocess.call(command) - print '[prerun] Made user {0} a sysadmin'.format(name) - - -if __name__ == '__main__': - - maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' - - if maintenance: - print '[prerun] Maintenance mode, skipping setup...' - else: - check_main_db_connection() - init_db() - update_plugins() - check_datastore_db_connection() - init_datastore_db() - check_solr_connection() - create_sysadmin() \ No newline at end of file diff --git a/ckan-base/2.8/setup/start_ckan.sh b/ckan-base/2.8/setup/start_ckan.sh deleted file mode 100755 index d1c6726..0000000 --- a/ckan-base/2.8/setup/start_ckan.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python prerun.py - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - -# Set the common uwsgi options -UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L" - -# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully -if [ $? -eq 0 ] -then - if [ "$PASSWORD_PROTECT" = true ] - then - if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] - then - # Generate htpasswd file for basicauth - htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi with basicauth - sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS - else - echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." - exit 1 - fi - else - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi - sudo -u ckan -EH uwsgi $UWSGI_OPTS - fi -else - echo "[prerun] failed...not starting CKAN." -fi diff --git a/ckan-base/2.8/setup/supervisor.worker.conf b/ckan-base/2.8/setup/supervisor.worker.conf deleted file mode 100755 index 0ca43ba..0000000 --- a/ckan-base/2.8/setup/supervisor.worker.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:ckan-worker] -command=paster --plugin=ckan jobs worker -c /srv/app/production.ini -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" - diff --git a/ckan-base/2.8/setup/supervisord.conf b/ckan-base/2.8/setup/supervisord.conf deleted file mode 100755 index a3f6671..0000000 --- a/ckan-base/2.8/setup/supervisord.conf +++ /dev/null @@ -1,23 +0,0 @@ -[unix_http_server] -file = /tmp/supervisor.sock -chmod = 0777 -chown = nobody:nogroup - -[supervisord] -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = info -pidfile = /tmp/supervisord.pid -nodaemon = true -umask = 022 -identifier = supervisor - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[include] -files = /etc/supervisord.d/*.conf diff --git a/ckan-base/2.8/setup/uwsgi.conf b/ckan-base/2.8/setup/uwsgi.conf deleted file mode 100755 index 6321d6d..0000000 --- a/ckan-base/2.8/setup/uwsgi.conf +++ /dev/null @@ -1,2 +0,0 @@ -[uwsgi] -route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd diff --git a/ckan-base/2.9/Dockerfile b/ckan-base/2.9/Dockerfile deleted file mode 100755 index 45b5aae..0000000 --- a/ckan-base/2.9/Dockerfile +++ /dev/null @@ -1,109 +0,0 @@ -FROM alpine:3.13 - -# Internal environment variables -ENV APP_DIR=/srv/app -ENV SRC_DIR=/srv/app/src -ENV CKAN_INI=${APP_DIR}/ckan.ini -ENV PIP_SRC=${SRC_DIR} -ENV CKAN_STORAGE_PATH=/var/lib/ckan -ENV GIT_URL=https://github.com/ckan/ckan.git -# CKAN version to build -ENV GIT_BRANCH=ckan-2.9.4 -# Customize these on the .env file if needed -ENV CKAN_SITE_URL=http://localhost:5000 -ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars - -WORKDIR ${APP_DIR} - -# Install necessary packages to run CKAN -RUN apk add --no-cache tzdata \ - git \ - gettext \ - postgresql-client \ - python3 \ - apache2-utils \ - libxml2 \ - libxslt \ - musl-dev \ - uwsgi-http \ - uwsgi-corerouter \ - uwsgi-python3 \ - py3-gevent \ - uwsgi-gevent \ - libmagic \ - curl \ - sudo && \ - # Packages to build CKAN requirements and plugins - apk add --no-cache --virtual .build-deps \ - postgresql-dev \ - gcc \ - make \ - g++ \ - autoconf \ - automake \ - libtool \ - python3-dev \ - py3-virtualenv \ - libxml2-dev \ - libxslt-dev \ - linux-headers - -# Create src directory (SRC_DIR) -RUN mkdir -p ${SRC_DIR} - -# Install pip -RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ - python3 ${SRC_DIR}/get-pip.py - -# Set up Python3 virtual environment -RUN cd ${APP_DIR} && \ - python3 -m venv ${APP_DIR} && \ - source ${APP_DIR}/bin/activate - -# Virtual environment binaries/scripts to be used first -ENV PATH=${APP_DIR}/bin:${PATH} - -# Install CKAN code -RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan - -# Install uwsgi -RUN pip3 install uwsgi - -# Complete CKAN install -RUN cd ${SRC_DIR}/ckan && \ - cp who.ini ${APP_DIR} && \ - pip3 install -r requirement-setuptools.txt && \ - pip3 install --no-binary :all: -r requirements.txt && \ - # Install CKAN envvars to support loading config from environment variables - pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars - -# Create and update CKAN config -RUN ckan generate config ${CKAN_INI} - -# Install and configure supervisor -RUN pip3 install supervisor && \ -mkdir /etc/supervisord.d - -# Copy all setup files -COPY 2.9/setup ${APP_DIR} -COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf -COPY 2.9/setup/supervisord.conf /etc/supervisord.conf - -# Create a local user and group to run the app -RUN addgroup -g 92 -S ckan && \ - adduser -u 92 -h /srv/app -H -D -S -G ckan ckan - -# Create local storage folder -RUN mkdir -p $CKAN_STORAGE_PATH && \ - chown -R ckan:ckan $CKAN_STORAGE_PATH - -# Create entrypoint directory for children image scripts -ONBUILD RUN mkdir /docker-entrypoint.d - -RUN chown ckan -R /srv/app - -EXPOSE 5000 - -HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 - -CMD ["/srv/app/start_ckan.sh"] \ No newline at end of file diff --git a/ckan-base/2.9/setup/ckan-uwsgi.ini b/ckan-base/2.9/setup/ckan-uwsgi.ini deleted file mode 100644 index 2361f36..0000000 --- a/ckan-base/2.9/setup/ckan-uwsgi.ini +++ /dev/null @@ -1,15 +0,0 @@ -[uwsgi] -http-socket = :5000 -uid = ckan -guid = ckan -plugins = python3 -wsgi-file = /srv/app/wsgi.py -virtualenv = /srv/app -module = wsgi:application -master = true -processes = 5 -pidfile = /tmp/%n.pid -harakiri = 50 -max-requests = 5000 -vacuum = true -callable = application diff --git a/ckan-base/2.9/setup/prerun.py b/ckan-base/2.9/setup/prerun.py deleted file mode 100755 index f8dd693..0000000 --- a/ckan-base/2.9/setup/prerun.py +++ /dev/null @@ -1,194 +0,0 @@ -import os -import sys -import subprocess -import psycopg2 -import urllib3 -import time -import re - -ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini') - -RETRY = 5 - -def update_plugins(): - - plugins = os.environ.get('CKAN__PLUGINS', '') - print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) - print(plugins) - cmd = ['ckan', 'config-tool', ckan_ini, - 'ckan.plugins = {}'.format(plugins)] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - print('[prerun] Plugins set.') - - -def check_main_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') - if not conn_str: - print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db') - return check_db_connection(conn_str, retry) - - -def check_datastore_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db') - return check_db_connection(conn_str, retry) - - -def check_db_connection(conn_str, retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print('[prerun] Giving up after 5 tries...') - sys.exit(1) - - try: - connection = psycopg2.connect(conn_str) - - except psycopg2.Error as e: - print(str(e)) - print('[prerun] Unable to connect to the database, waiting...') - time.sleep(10) - check_db_connection(conn_str, retry=retry - 1) - else: - connection.close() - - -def check_solr_connection(retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print('[prerun] Giving up after 5 tries...') - sys.exit(1) - - url = os.environ.get('CKAN_SOLR_URL', '') - search_url = '{url}/select/?q=*&wt=json'.format(url=url) - http = urllib3.PoolManager() - try: - r = http.request('GET', search_url) - except urllib3.exceptions.ConnectionError as e: - print(str(e)) - print('[prerun] Unable to connect to solr, waiting...') - time.sleep(10) - check_solr_connection(retry=retry - 1) - else: - print('[prerun] Connection Status from SOLR is ', (r.status)) - -def init_db(): - - db_command = ['ckan', '-c', ckan_ini, - 'db', 'init'] - print('[prerun] Initializing or upgrading db - start') - try: - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print('[prerun] Initializing or upgrading db - end') - except subprocess.CalledProcessError as e: - if 'OperationalError' in e.output: - print(e.output) - print('[prerun] Database not ready, waiting a bit before exit...') - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - - -def init_datastore_db(): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print('[prerun] Skipping datastore initialization') - return - - datastore_perms_command = ['ckan', '-c', ckan_ini, - 'datastore', 'set-permissions'] - - connection = psycopg2.connect(conn_str) - cursor = connection.cursor() - - print('[prerun] Initializing datastore db - start') - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, - stdout=subprocess.PIPE) - - perms_sql = datastore_perms.stdout.read().decode('utf-8') - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print(notice) - - connection.commit() - - print('[prerun] Initializing datastore db - end') - print(datastore_perms.stdout.read().decode('utf-8')) - except psycopg2.Error as e: - print('[prerun] Could not initialize datastore') - print(str(e)) - - except subprocess.CalledProcessError as e: - if 'OperationalError' in e.output: - print(e.output) - print('[prerun] Database not ready, waiting a bit before exit...') - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - name = os.environ.get('CKAN_SYSADMIN_NAME') - password = os.environ.get('CKAN_SYSADMIN_PASSWORD') - email = os.environ.get('CKAN_SYSADMIN_EMAIL') - - if name and password and email: - - # Check if user exists - command = ['ckan', '-c', ckan_ini, 'user', 'show', name,] - - out = subprocess.check_output(command) - if 'User:None' not in re.sub(r'\s', '', out.decode()): - print('[prerun] Sysadmin user exists, skipping creation') - return - - # Create user - command = ['ckan', '-c', ckan_ini, 'user', 'add', - name, - 'password=' + password, - 'email=' + email] - - subprocess.call(command) - print('[prerun] Created user {0}'.format(name)) - - # Make it sysadmin - command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', - name] - - subprocess.call(command) - print('[prerun] Made user {0} a sysadmin'.format(name)) - - -if __name__ == '__main__': - - maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' - - if maintenance: - print('[prerun] Maintenance mode, skipping setup...') - else: - check_main_db_connection() - init_db() - update_plugins() - check_datastore_db_connection() - init_datastore_db() - check_solr_connection() - create_sysadmin() diff --git a/ckan-base/2.9/setup/start_ckan.sh b/ckan-base/2.9/setup/start_ckan.sh deleted file mode 100755 index 5a119e6..0000000 --- a/ckan-base/2.9/setup/start_ckan.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python3 prerun.py - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - - -# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully -if [ $? -eq 0 ] -then - if [ "$PASSWORD_PROTECT" = true ] - then - if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] - then - # Generate htpasswd file for basicauth - htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi with basicauth - sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini - else - echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." - exit 1 - fi - else - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi - sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini - fi -else - echo "[prerun] failed...not starting CKAN." -fi - diff --git a/ckan-base/2.9/setup/supervisor.worker.conf b/ckan-base/2.9/setup/supervisor.worker.conf deleted file mode 100644 index 9d46f37..0000000 --- a/ckan-base/2.9/setup/supervisor.worker.conf +++ /dev/null @@ -1,12 +0,0 @@ -[program:ckan-worker] -command=ckan -c /srv/app/ckan.ini jobs worker -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" \ No newline at end of file diff --git a/ckan-base/2.9/setup/supervisord.conf b/ckan-base/2.9/setup/supervisord.conf deleted file mode 100644 index 052dbc5..0000000 --- a/ckan-base/2.9/setup/supervisord.conf +++ /dev/null @@ -1,23 +0,0 @@ -[unix_http_server] -file = /tmp/supervisor.sock -chmod = 0777 -chown = nobody:nogroup - -[supervisord] -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = info -pidfile = /tmp/supervisord.pid -nodaemon = true -umask = 022 -identifier = supervisor - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[include] -files = /etc/supervisord.d/*.conf \ No newline at end of file diff --git a/ckan-base/2.9/setup/wsgi.py b/ckan-base/2.9/setup/wsgi.py deleted file mode 100644 index b37d80e..0000000 --- a/ckan-base/2.9/setup/wsgi.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -from ckan.config.middleware import make_app -from ckan.cli import CKANConfigLoader -from logging.config import fileConfig as loggingFileConfig -config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') -abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) -loggingFileConfig(config_filepath) -config = CKANConfigLoader(config_filepath).get_config() -application = make_app(config) diff --git a/ckan-dev/2.7/Dockerfile b/ckan-dev/2.7/Dockerfile deleted file mode 100755 index a421a59..0000000 --- a/ckan-dev/2.7/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ckan/ckan-base:testing-only.2.7 - -LABEL maintainer="brett@kowh.ai" - -ENV APP_DIR=/srv/app -ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions - -# Install packages needed by the dev requirements -RUN apk add --no-cache libffi-dev - -# Install CKAN dev requirements -RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt - -# Create folder for local extensions sources -RUN mkdir $SRC_EXTENSIONS_DIR - -COPY setup/start_ckan_development.sh ${APP_DIR} - - -CMD ["/srv/app/start_ckan_development.sh"] diff --git a/ckan-dev/2.8/Dockerfile b/ckan-dev/2.8/Dockerfile deleted file mode 100755 index 6142a8d..0000000 --- a/ckan-dev/2.8/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ckan/ckan-base:testing-only.2.8 - -LABEL maintainer="brett@kowh.ai" - -ENV APP_DIR=/srv/app -ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions - -# Install packages needed by the dev requirements -RUN apk add --no-cache libffi-dev - -# Install CKAN dev requirements -RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt - -# Create folder for local extensions sources -RUN mkdir $SRC_EXTENSIONS_DIR - -COPY setup/start_ckan_development.sh ${APP_DIR} - - -CMD ["/srv/app/start_ckan_development.sh"] diff --git a/ckan-dev/2.9/Dockerfile b/ckan-dev/2.9/Dockerfile deleted file mode 100755 index fc170ee..0000000 --- a/ckan-dev/2.9/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM ckan/ckan-base:testing-only.2.9 - -LABEL maintainer="brett@kowh.ai" - -ENV APP_DIR=/srv/app -ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions - -# Install packages needed by the dev requirements -RUN apk add --no-cache libffi-dev - -# Set up Python3 virtual environment -RUN cd ${APP_DIR} && \ - source ${APP_DIR}/bin/activate - -# Virtual environment binaries/scripts to be used first -ENV PATH=${APP_DIR}/bin:${PATH} - -# Install CKAN dev requirements -# Will need to change this eventually - when CKAN 2.9 is out -# wget https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt -# RUN pip3 install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt -RUN pip3 install -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt - -# Create folder for local extensions sources -RUN mkdir $SRC_EXTENSIONS_DIR - -COPY setup/start_ckan_development.sh ${APP_DIR} - -CMD ["/srv/app/start_ckan_development.sh"] diff --git a/ckan-dev/setup/start_ckan_development.sh b/ckan-dev/setup/start_ckan_development.sh deleted file mode 100755 index 331266c..0000000 --- a/ckan-dev/setup/start_ckan_development.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# Install any local extensions in the src_extensions volume -echo "Looking for local extensions to install..." -echo "Extension dir contents:" -ls -la $SRC_EXTENSIONS_DIR -for i in $SRC_EXTENSIONS_DIR/* -do - if [ -d $i ]; - then - - if [ -f $i/pip-requirements.txt ]; - then - pip install -r $i/pip-requirements.txt - echo "Found requirements file in $i" - fi - if [ -f $i/requirements.txt ]; - then - pip install -r $i/requirements.txt - echo "Found requirements file in $i" - fi - if [ -f $i/dev-requirements.txt ]; - then - pip install -r $i/dev-requirements.txt - echo "Found dev-requirements file in $i" - fi - if [ -f $i/setup.py ]; - then - cd $i - python $i/setup.py develop - echo "Found setup.py file in $i" - cd $APP_DIR - fi - - # Point `use` in test.ini to location of `test-core.ini` - if [ -f $i/test.ini ]; - then - echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" - paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" - fi - fi -done - -# Set debug to true -echo "Enabling debug mode" -ckan config-tool $CKAN_INI -s DEFAULT "debug = true" - -# Update the plugins setting in the ini file with the values defined in the env var -echo "Loading the following plugins: $CKAN__PLUGINS" -ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" - -# Update test-core.ini DB, SOLR & Redis settings -echo "Loading test settings into test-core.ini" -ckan config-tool $SRC_DIR/ckan/test-core.ini \ - "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ - "ckan.datstore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ - "ckan.datstore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ - "solr_url = $TEST_CKAN_SOLR_URL" \ - "ckan.redis_url = $TEST_CKAN_REDIS_URL" - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python prerun.py - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - -# Start supervisord -supervisord --configuration /etc/supervisord.conf & - -# Start the development server with automatic reload -# Check the --reloader options sudo -u ckan -EH ckan -c $CKAN_INI run --reloader -sudo -u ckan -EH ckan -c $CKAN_INI run diff --git a/docker-compose.yml b/docker-compose.yml index c99e3d4..b2f5aa1 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,5 @@ version: "3" -volumes: - pg_data: - solr_data: - ckan_storage: - services: nginx: @@ -16,7 +11,7 @@ services: - ckan ports: - "0.0.0.0:80:80" - + ckan: container_name: ckan build: @@ -34,13 +29,18 @@ services: ports: - "0.0.0.0:${CKAN_PORT}:5000" volumes: - - ckan_storage:/var/lib/ckan + - ckan_storage:/var/lib/ckan + + datapusher: + container_name: datapusher + image: kowhai/datapusher:0.0.17 + ports: + - "8800:8800" db: container_name: db build: - context: . - dockerfile: postgresql/Dockerfile + context: postgresql/ args: - DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -50,31 +50,19 @@ services: - PGDATA=/var/lib/postgresql/data/db volumes: - pg_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD", "pg_isready", "-U", "ckan"] - - datapusher: - container_name: datapusher - build: - context: datapusher - ports: - - "8800:8800" - environment: - - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} - - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} - - CKAN__DATAPUSHER__CALLBACK_URL_BASE=${CKAN__DATAPUSHER__CALLBACK_URL_BASE} - + solr: container_name: solr build: - context: . - dockerfile: solr/Dockerfile - args: - - SOLR_VERSION=${SOLR_VERSION} - - CKAN_VERSION=${CKAN_VERSION} + context: solr/ volumes: - - solr_data:/opt/solr/server/solr/ckan/data + - solr_data:/opt/solr/server/solr/ckan/data/index redis: container_name: redis - image: redis:${REDIS_VERSION} + image: redis:alpine + +volumes: + ckan_storage: + pg_data: + solr_data: diff --git a/docker-entrypoint-initdb.d/10_create_datastore.sql b/docker-entrypoint-initdb.d/10_create_datastore.sql deleted file mode 100755 index 87529ce..0000000 --- a/docker-entrypoint-initdb.d/10_create_datastore.sql +++ /dev/null @@ -1,4 +0,0 @@ -\set datastore_ro_password '\'' `echo $DATASTORE_READONLY_PASSWORD` '\'' - -CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD :datastore_ro_password; -CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; \ No newline at end of file diff --git a/docker-entrypoint-initdb.d/20_setup_test_databases.sql b/docker-entrypoint-initdb.d/20_setup_test_databases.sql deleted file mode 100755 index 61366d2..0000000 --- a/docker-entrypoint-initdb.d/20_setup_test_databases.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8'; -CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8'; \ No newline at end of file diff --git a/index.html b/index.html deleted file mode 100644 index 561b63b..0000000 --- a/index.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - CKAN Docker NGINX landing page - - - - -

- CKAN Docker NGINX landing page -

- - - \ No newline at end of file diff --git a/notes.txt b/notes.txt deleted file mode 100644 index dccc9b1..0000000 --- a/notes.txt +++ /dev/null @@ -1,69 +0,0 @@ - -Environment variables are in the .env file and loaded into the docker-compose.yml file at runtime. -The Dockerfile will need an ARG line also - -List all env variables and decide where they go -.env -env-ckan -ARG -ENV - -NB: .env variables can be referenced in the docker-compose.yml file (environment: part of the service) - The docker-compose.yml file can pass these variables to the Dockerfile as ARG parameters ---------------------------- from OKFN --------------------------- -# DB image settings -POSTGRES_PASSWORD=ckan -DATASTORE_READONLY_PASSWORD=datastore - -# Basic -CKAN_SITE_ID=default -CKAN_SITE_URL=http://ckan:5000 -CKAN_PORT=5000 -CKAN_SYSADMIN_NAME=ckan_admin -CKAN_SYSADMIN_PASSWORD=test1234 -CKAN_SYSADMIN_EMAIL=your_email@example.com -TZ=UTC - -# Database connections (TODO: avoid duplication) -CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan -CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore - -# Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test - -# Other services connections -CKAN_SOLR_URL=http://solr:8983/solr/ckan -CKAN_REDIS_URL=redis://redis:6379/1 -CKAN_DATAPUSHER_URL=http://datapusher:8800 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 - -TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan -TEST_CKAN_REDIS_URL=redis://redis:6379/1 - -# Core settings -CKAN__STORAGE_PATH=/var/lib/ckan - -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost - -# Extensions - -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 - - -docker-compose.yml -DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD} -POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD} -POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -PGDATA=/var/lib/postgresql/data/db \ No newline at end of file diff --git a/patches/00_patch_sql_url.patch b/patches/00_patch_sql_url.patch deleted file mode 100644 index 46a7135..0000000 --- a/patches/00_patch_sql_url.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ckan/ckan/model/__init__.py 2021-02-16 14:47:06.168327441 +0100 -+++ ckan/ckan/model/__init__.py 2021-02-16 14:48:00.740780218 +0100 -@@ -266,7 +266,7 @@ - self.reset_alembic_output() - alembic_config = AlembicConfig(self._alembic_ini) - alembic_config.set_main_option( -- "sqlalchemy.url", str(self.metadata.bind.url) -+ "sqlalchemy.url", str(self.metadata.bind.url).replace('%', '%%') - ) - try: - sqlalchemy_migrate_version = self.metadata.bind.execute( diff --git a/patches/01_patch_resource_replace_upload.patch b/patches/01_patch_resource_replace_upload.patch deleted file mode 100644 index 3f11fac..0000000 --- a/patches/01_patch_resource_replace_upload.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ckan/ckan/logic/action/update.py 2021-02-17 16:46:55.673578728 +0100 -+++ ckan/ckan/logic/action/update-edit.py 2021-02-17 16:47:28.905879170 +0100 -@@ -929,7 +929,7 @@ - - ''' - model = context['model'] -- session = model.Session -+ session = model.meta.create_local_session() - context['session'] = session - - user = context['user'] diff --git a/patches/02_patch_postgres_username_split.patch b/patches/02_patch_postgres_username_split.patch deleted file mode 100644 index 780b432..0000000 --- a/patches/02_patch_postgres_username_split.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ckan/ckanext/datastore/backend/postgres.py 2021-02-18 11:01:56.692267462 +0100 -+++ ckan/ckanext/datastore/backend/postgres-patch.py 2021-02-18 13:45:16.033193435 +0100 -@@ -1690,7 +1690,7 @@ - read only user. - ''' - write_connection = self._get_write_engine().connect() -- read_connection_user = sa_url.make_url(self.read_url).username -+ read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0] - - drop_foo_sql = u'DROP TABLE IF EXISTS _foo' - diff --git a/scripts/apply_ckan_patches.sh b/scripts/apply_ckan_patches.sh deleted file mode 100755 index a7bceb9..0000000 --- a/scripts/apply_ckan_patches.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -shopt -s nullglob -for patch in patches/*.patch; do - /usr/bin/patch -p0 -i $patch -done diff --git a/setup/ckan-uwsgi.ini b/setup/ckan-uwsgi.ini deleted file mode 100644 index 2361f36..0000000 --- a/setup/ckan-uwsgi.ini +++ /dev/null @@ -1,15 +0,0 @@ -[uwsgi] -http-socket = :5000 -uid = ckan -guid = ckan -plugins = python3 -wsgi-file = /srv/app/wsgi.py -virtualenv = /srv/app -module = wsgi:application -master = true -processes = 5 -pidfile = /tmp/%n.pid -harakiri = 50 -max-requests = 5000 -vacuum = true -callable = application diff --git a/setup/datapusher-uwsgi.ini b/setup/datapusher-uwsgi.ini deleted file mode 100644 index 4c8274a..0000000 --- a/setup/datapusher-uwsgi.ini +++ /dev/null @@ -1,14 +0,0 @@ -[uwsgi] -http = 0.0.0.0:8800 -uid = www-data -guid = www-data -wsgi-file = /srv/app/datapusher.wsgi -master = true -socket = /tmp/uwsgi.sock -plugins = http,python -pidfile = /tmp/%n.pid -harakiri = 50 -max-requests = 5000 -vacuum = true -callable = application -buffer-size = 32768 \ No newline at end of file diff --git a/setup/datapusher_settings.py b/setup/datapusher_settings.py deleted file mode 100644 index d6ee97a..0000000 --- a/setup/datapusher_settings.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import uuid - -DEBUG = False -TESTING = False -SECRET_KEY = str(uuid.uuid4()) -USERNAME = str(uuid.uuid4()) -PASSWORD = str(uuid.uuid4()) - -NAME = 'datapusher' - -# Webserver host and port - -HOST = os.environ.get('DATAPUSHER_HOST', '0.0.0.0') -PORT = os.environ.get('DATAPUSHER_PORT', 8800) - -# Database - -SQLALCHEMY_DATABASE_URI = os.environ.get('DATAPUSHER_SQLALCHEMY_DATABASE_URI', 'sqlite:////tmp/job_store.db') - -# Download and streaming settings - -MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '1024000')) -CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384')) -CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250')) -DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30')) - -# Verify SSL -SSL_VERIFY = os.environ.get('DATAPUSHER_SSL_VERIFY', True) - -# logging -#LOG_FILE = '/tmp/ckan_service.log' -STDERR = True - -# Rewrite resource URL's when ckan callback url base is used -REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', True) -REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/') diff --git a/setup/nginx.conf b/setup/nginx.conf deleted file mode 100644 index 7f06cea..0000000 --- a/setup/nginx.conf +++ /dev/null @@ -1,34 +0,0 @@ - -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - - include /etc/nginx/sites-enabled/*.conf; -} - - diff --git a/setup/prerun.py b/setup/prerun.py deleted file mode 100755 index f8dd693..0000000 --- a/setup/prerun.py +++ /dev/null @@ -1,194 +0,0 @@ -import os -import sys -import subprocess -import psycopg2 -import urllib3 -import time -import re - -ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini') - -RETRY = 5 - -def update_plugins(): - - plugins = os.environ.get('CKAN__PLUGINS', '') - print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) - print(plugins) - cmd = ['ckan', 'config-tool', ckan_ini, - 'ckan.plugins = {}'.format(plugins)] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - print('[prerun] Plugins set.') - - -def check_main_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') - if not conn_str: - print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db') - return check_db_connection(conn_str, retry) - - -def check_datastore_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db') - return check_db_connection(conn_str, retry) - - -def check_db_connection(conn_str, retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print('[prerun] Giving up after 5 tries...') - sys.exit(1) - - try: - connection = psycopg2.connect(conn_str) - - except psycopg2.Error as e: - print(str(e)) - print('[prerun] Unable to connect to the database, waiting...') - time.sleep(10) - check_db_connection(conn_str, retry=retry - 1) - else: - connection.close() - - -def check_solr_connection(retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print('[prerun] Giving up after 5 tries...') - sys.exit(1) - - url = os.environ.get('CKAN_SOLR_URL', '') - search_url = '{url}/select/?q=*&wt=json'.format(url=url) - http = urllib3.PoolManager() - try: - r = http.request('GET', search_url) - except urllib3.exceptions.ConnectionError as e: - print(str(e)) - print('[prerun] Unable to connect to solr, waiting...') - time.sleep(10) - check_solr_connection(retry=retry - 1) - else: - print('[prerun] Connection Status from SOLR is ', (r.status)) - -def init_db(): - - db_command = ['ckan', '-c', ckan_ini, - 'db', 'init'] - print('[prerun] Initializing or upgrading db - start') - try: - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print('[prerun] Initializing or upgrading db - end') - except subprocess.CalledProcessError as e: - if 'OperationalError' in e.output: - print(e.output) - print('[prerun] Database not ready, waiting a bit before exit...') - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - - -def init_datastore_db(): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print('[prerun] Skipping datastore initialization') - return - - datastore_perms_command = ['ckan', '-c', ckan_ini, - 'datastore', 'set-permissions'] - - connection = psycopg2.connect(conn_str) - cursor = connection.cursor() - - print('[prerun] Initializing datastore db - start') - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, - stdout=subprocess.PIPE) - - perms_sql = datastore_perms.stdout.read().decode('utf-8') - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print(notice) - - connection.commit() - - print('[prerun] Initializing datastore db - end') - print(datastore_perms.stdout.read().decode('utf-8')) - except psycopg2.Error as e: - print('[prerun] Could not initialize datastore') - print(str(e)) - - except subprocess.CalledProcessError as e: - if 'OperationalError' in e.output: - print(e.output) - print('[prerun] Database not ready, waiting a bit before exit...') - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - name = os.environ.get('CKAN_SYSADMIN_NAME') - password = os.environ.get('CKAN_SYSADMIN_PASSWORD') - email = os.environ.get('CKAN_SYSADMIN_EMAIL') - - if name and password and email: - - # Check if user exists - command = ['ckan', '-c', ckan_ini, 'user', 'show', name,] - - out = subprocess.check_output(command) - if 'User:None' not in re.sub(r'\s', '', out.decode()): - print('[prerun] Sysadmin user exists, skipping creation') - return - - # Create user - command = ['ckan', '-c', ckan_ini, 'user', 'add', - name, - 'password=' + password, - 'email=' + email] - - subprocess.call(command) - print('[prerun] Created user {0}'.format(name)) - - # Make it sysadmin - command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', - name] - - subprocess.call(command) - print('[prerun] Made user {0} a sysadmin'.format(name)) - - -if __name__ == '__main__': - - maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' - - if maintenance: - print('[prerun] Maintenance mode, skipping setup...') - else: - check_main_db_connection() - init_db() - update_plugins() - check_datastore_db_connection() - init_datastore_db() - check_solr_connection() - create_sysadmin() diff --git a/setup/sites-available/ckan.conf b/setup/sites-available/ckan.conf deleted file mode 100644 index 7ffc6bb..0000000 --- a/setup/sites-available/ckan.conf +++ /dev/null @@ -1,18 +0,0 @@ -proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m; -proxy_temp_path /tmp/nginx_proxy 1 2; - -server { - client_max_body_size 100M; - location / { - proxy_pass http://ckan:5000/; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header Host $host; - proxy_cache cache; - proxy_cache_bypass $cookie_auth_tkt; - proxy_no_cache $cookie_auth_tkt; - proxy_cache_valid 30m; - proxy_cache_key $host$scheme$proxy_host$request_uri; - # In emergency comment out line to force caching - # proxy_ignore_headers X-Accel-Expires Expires Cache-Control; - } -} \ No newline at end of file diff --git a/setup/start_ckan.sh b/setup/start_ckan.sh deleted file mode 100755 index 5a119e6..0000000 --- a/setup/start_ckan.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python3 prerun.py - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - - -# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully -if [ $? -eq 0 ] -then - if [ "$PASSWORD_PROTECT" = true ] - then - if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] - then - # Generate htpasswd file for basicauth - htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi with basicauth - sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini - else - echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." - exit 1 - fi - else - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi - sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini - fi -else - echo "[prerun] failed...not starting CKAN." -fi - diff --git a/setup/start_datapusher.sh b/setup/start_datapusher.sh deleted file mode 100755 index 3b78d19..0000000 --- a/setup/start_datapusher.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -echo "[start_datapusher.sh] Starting supervisord." -# Start supervisord -supervisord --configuration /etc/supervisord.conf diff --git a/setup/supervisor.uwsgi.conf b/setup/supervisor.uwsgi.conf deleted file mode 100644 index 67a6c37..0000000 --- a/setup/supervisor.uwsgi.conf +++ /dev/null @@ -1,12 +0,0 @@ -[program:datapusher-uwsgi] -command=/usr/sbin/uwsgi -i /srv/app/datapusher-uwsgi.ini -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=www-data -environment=HOME="/srv/app",USER="www-data" \ No newline at end of file diff --git a/setup/supervisor.worker.conf b/setup/supervisor.worker.conf deleted file mode 100644 index 9d46f37..0000000 --- a/setup/supervisor.worker.conf +++ /dev/null @@ -1,12 +0,0 @@ -[program:ckan-worker] -command=ckan -c /srv/app/ckan.ini jobs worker -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" \ No newline at end of file diff --git a/setup/supervisord.conf b/setup/supervisord.conf deleted file mode 100644 index f21c89c..0000000 --- a/setup/supervisord.conf +++ /dev/null @@ -1,23 +0,0 @@ -[unix_http_server] -file = /tmp/supervisor.sock -chmod = 0777 -chown = nobody:nogroup - -[supervisord] -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = trace -pidfile = /tmp/supervisord.pid -nodaemon = true -umask = 022 -identifier = supervisor - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[include] -files = /etc/supervisord.d/*.conf \ No newline at end of file diff --git a/setup/wsgi.py b/setup/wsgi.py deleted file mode 100644 index b37d80e..0000000 --- a/setup/wsgi.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -from ckan.config.middleware import make_app -from ckan.cli import CKANConfigLoader -from logging.config import fileConfig as loggingFileConfig -config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') -abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) -loggingFileConfig(config_filepath) -config = CKANConfigLoader(config_filepath).get_config() -application = make_app(config) diff --git a/solr/Dockerfile b/solr/Dockerfile index 1e0c650..8922342 100755 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -2,10 +2,9 @@ FROM solr:6.6.6 # Enviroment variables ENV SOLR_CORE ckan - -# Build Arguments -ARG CKAN_VERSION -ARG SOLR_VERSION +ENV SOLR_VERSION 6.6.6 +ENV CKAN_VERSION 2.9.1 +###TODO!!! CKAN_VERSION to be passed in as an ARG # root user for initial config USER root @@ -16,14 +15,14 @@ RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \ mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index # Add files -COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml -ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ -/opt/solr/server/solr/${SOLR_CORE}/conf/ +ADD solrconfig.xml \ +https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +/opt/solr/server/solr/$SOLR_CORE/conf/ # Create core.properties RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties @@ -33,4 +32,3 @@ RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE} # non-root user for runtime USER ${SOLR_USER}:${SOLR_USER} - diff --git a/solr/solrconfig-2.9.3.xml b/solr/solrconfig.xml old mode 100644 new mode 100755 similarity index 99% rename from solr/solrconfig-2.9.3.xml rename to solr/solrconfig.xml index 5ddc4c4..8a3eade --- a/solr/solrconfig-2.9.3.xml +++ b/solr/solrconfig.xml @@ -342,4 +342,4 @@ 5 - \ No newline at end of file + diff --git a/todo.txt b/todo.txt deleted file mode 100644 index d5d6960..0000000 --- a/todo.txt +++ /dev/null @@ -1,12 +0,0 @@ -Get the NGINX container running -Add the redis container -Add the solr container -Add the db container -Get a datapusher container working with supervisord -Test the datapusher container works -Take out ARG's from docker-compose.yml is not needed -Test with new versions of CKAN (ie: if problems with Python 3.9) -Create (and push) the datapusher image -Create the ckan container -Create the ckanext container -