From 7e244290a5d436ecf1ac00a51051825d550f1608 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 12 Dec 2022 17:02:22 +0100 Subject: [PATCH] Remove images directory + TODO's file --- TODO's.txt | 39 ---- images/README | 13 -- images/ckan-base/2.9/Dockerfile | 103 ----------- images/ckan-base/2.9/setup/ckan-uwsgi.ini | 15 -- images/ckan-base/2.9/setup/prerun.py | 194 -------------------- images/ckan-base/2.9/setup/start_ckan.sh | 39 ---- images/ckan-base/2.9/setup/supervisord.conf | 23 --- images/ckan-base/2.9/setup/wsgi.py | 9 - images/datapusher/Dockerfile | 71 ------- 9 files changed, 506 deletions(-) delete mode 100644 TODO's.txt delete mode 100755 images/README delete mode 100755 images/ckan-base/2.9/Dockerfile delete mode 100644 images/ckan-base/2.9/setup/ckan-uwsgi.ini delete mode 100755 images/ckan-base/2.9/setup/prerun.py delete mode 100755 images/ckan-base/2.9/setup/start_ckan.sh delete mode 100644 images/ckan-base/2.9/setup/supervisord.conf delete mode 100644 images/ckan-base/2.9/setup/wsgi.py delete mode 100755 images/datapusher/Dockerfile diff --git a/TODO's.txt b/TODO's.txt deleted file mode 100644 index 5d492e0..0000000 --- a/TODO's.txt +++ /dev/null @@ -1,39 +0,0 @@ -# (from) July 7 2022 -# This repo will be actively worked on from now. -# This file will be a ToDo/bits'n'bobs list of things to take care of - -ckan/ckan-docker-base: For the base images Dockerfiles (prod and dev) and related scripts -ckan/ckan-docker: For the project-oriented image template (prod and dev). Patching only done in Dev ### This repo! - -All the other images should live in separate repos - -1. Solr - use ckan-solr (https://github.com/ckan/ckan-solr) -2. PostgreSQL - use current method (base image: postgres:12-alpine from DockerHub, enhanced in a Dockerfile) - ### This may change to be more like Solr though -3. Redis - use current method (DockerHub image: redis:${REDIS_VERSION} specified as a compose service in the compose file) - latest image to used is redis:6 -4. nginx - use a late(ish) nginx image from DockerHub, enhanced in a Dockerfile) -5. DataPusher - built from the actual datapusher repo (https://github.com/ckan/datapusher) -6. CKAN - built from the ckan/ckan-base:2.9.5 base image (which is built from the ckan/ckan-docker-base repo). Include a .dev option -7. CKAN Worker - add new (ckan worker) container in the compose setup...maybe have it commented out initially - -Versions 2.9 and 2.10 (when it's out) only. Plan the repo layout for having multiple versions - OKFN could used as an example - -NB: Had to update the prerun.py script as it was failing on check_solr_connection - - -### ToDo (remaining things to think about and/or 'to fix') ### - -1. DataPusher - needed to use a custom requirements.txt as the official didn't work (see https://github.com/ckan/datapusher/pull/251) -2. Use Multi-Stage images defined in Dockerfile - get the CKAN images down to bare-bones...for security reasons really - General Dockerfile flow of stages: - - base: all prod dependencies, no code yet - - dev, from base: all dev dependencies, no code yet (in dev, source code is bind-mounted anyway) - - source, from base: add code - - test/audit, from source: then COPY --from=dev for dev dependencies, then run tests. Optionally, audit and lint code (if you don't do it on git push already). - - prod, from source: no change from source stage, but listed last so in case a stage isn't targeted, the builder will default to this stage - Also check out https://github.com/ckan/ckan/pull/4635 for Francesco's test stuff -3. CKAN Worker (maybe) -4. Implement SSL for the nginx container or include a howto in the docs ### NB: This has been done! albeit with a self-signed cert - - diff --git a/images/README b/images/README deleted file mode 100755 index ac994c2..0000000 --- a/images/README +++ /dev/null @@ -1,13 +0,0 @@ -Build the images from this directory using: - cd ckan-base/ - docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile . - docker build -t ckan/ckan-base:testing-only.2.8 -f 2.8/Dockerfile . - docker build -t ckan/ckan-base:testing-only.2.7 -f 2.7/Dockerfile . - -Login to Dockerhub -docker login --username=kowhai -(will then ask for a password) - -Push image to (ckan) Dockerhub -docker push ckan/ckan-base:testing-only.2.9 - diff --git a/images/ckan-base/2.9/Dockerfile b/images/ckan-base/2.9/Dockerfile deleted file mode 100755 index 80f9cc9..0000000 --- a/images/ckan-base/2.9/Dockerfile +++ /dev/null @@ -1,103 +0,0 @@ -FROM alpine:3.13 - -# Internal environment variables -ENV APP_DIR=/srv/app -ENV SRC_DIR=/srv/app/src -ENV CKAN_INI=${APP_DIR}/ckan.ini -ENV PIP_SRC=${SRC_DIR} -ENV CKAN_STORAGE_PATH=/var/lib/ckan -ENV GIT_URL=https://github.com/ckan/ckan.git -# CKAN version to build -ENV GIT_BRANCH=ckan-2.9.5 -# Customize these on the .env file if needed -ENV CKAN_SITE_URL=http://localhost:5000 -ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars - -WORKDIR ${APP_DIR} - -# Install necessary packages to run CKAN -RUN apk add --no-cache tzdata \ - git \ - gettext \ - postgresql-client \ - python3 \ - apache2-utils \ - libxml2 \ - libxslt \ - musl-dev \ - uwsgi-http \ - uwsgi-corerouter \ - uwsgi-python3 \ - py3-gevent \ - uwsgi-gevent \ - libmagic \ - curl \ - sudo && \ - # Packages to build CKAN requirements and plugins - apk add --no-cache --virtual .build-deps \ - postgresql-dev \ - gcc \ - make \ - g++ \ - autoconf \ - automake \ - libtool \ - python3-dev \ - py3-virtualenv \ - libxml2-dev \ - libxslt-dev \ - linux-headers - -# Create SRC_DIR and install pip -RUN mkdir -p ${SRC_DIR} && \ - # Install pip - curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ - python3 ${SRC_DIR}/get-pip.py && \ - rm -rf ${SRC_DIR}/get-pip.py - -# Set up Python3 virtual environment -RUN cd ${APP_DIR} && \ - python3 -m venv ${APP_DIR} && \ - source ${APP_DIR}/bin/activate - -# Virtual environment binaries/scripts to be used first -ENV PATH=${APP_DIR}/bin:${PATH} - -# Install CKAN, uwsgi, ckanext-envvars and generate CKAN config file -RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ - pip3 install uwsgi && \ - cd ${SRC_DIR}/ckan && \ - cp who.ini ${APP_DIR} && \ - pip3 install -r requirement-setuptools.txt && \ - pip3 install --no-binary :all: -r requirements.txt && \ - # Install CKAN envvars to support loading config from environment variables - pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \ - # Create and update CKAN config - ckan generate config ${CKAN_INI} - -# Install and configure supervisor -RUN pip3 install supervisor && \ -mkdir /etc/supervisord.d - -# Copy setup files -COPY 2.9/setup ${APP_DIR} -RUN mv ${APP_DIR}/supervisord.conf /etc/supervisord.conf - -# Create a local user and group to run the app -RUN addgroup -g 92 -S ckan && \ - adduser -u 92 -h /srv/app -H -D -S -G ckan ckan - -# Create local storage folder -RUN mkdir -p $CKAN_STORAGE_PATH && \ - chown -R ckan:ckan $CKAN_STORAGE_PATH - -# Create entrypoint directory for children image scripts -ONBUILD RUN mkdir /docker-entrypoint.d - -RUN chown ckan -R /srv/app - -EXPOSE 5000 - -HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 - -CMD ["/srv/app/start_ckan.sh"] \ No newline at end of file diff --git a/images/ckan-base/2.9/setup/ckan-uwsgi.ini b/images/ckan-base/2.9/setup/ckan-uwsgi.ini deleted file mode 100644 index 2361f36..0000000 --- a/images/ckan-base/2.9/setup/ckan-uwsgi.ini +++ /dev/null @@ -1,15 +0,0 @@ -[uwsgi] -http-socket = :5000 -uid = ckan -guid = ckan -plugins = python3 -wsgi-file = /srv/app/wsgi.py -virtualenv = /srv/app -module = wsgi:application -master = true -processes = 5 -pidfile = /tmp/%n.pid -harakiri = 50 -max-requests = 5000 -vacuum = true -callable = application diff --git a/images/ckan-base/2.9/setup/prerun.py b/images/ckan-base/2.9/setup/prerun.py deleted file mode 100755 index f8dd693..0000000 --- a/images/ckan-base/2.9/setup/prerun.py +++ /dev/null @@ -1,194 +0,0 @@ -import os -import sys -import subprocess -import psycopg2 -import urllib3 -import time -import re - -ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini') - -RETRY = 5 - -def update_plugins(): - - plugins = os.environ.get('CKAN__PLUGINS', '') - print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) - print(plugins) - cmd = ['ckan', 'config-tool', ckan_ini, - 'ckan.plugins = {}'.format(plugins)] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - print('[prerun] Plugins set.') - - -def check_main_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') - if not conn_str: - print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db') - return check_db_connection(conn_str, retry) - - -def check_datastore_db_connection(retry=None): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db') - return check_db_connection(conn_str, retry) - - -def check_db_connection(conn_str, retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print('[prerun] Giving up after 5 tries...') - sys.exit(1) - - try: - connection = psycopg2.connect(conn_str) - - except psycopg2.Error as e: - print(str(e)) - print('[prerun] Unable to connect to the database, waiting...') - time.sleep(10) - check_db_connection(conn_str, retry=retry - 1) - else: - connection.close() - - -def check_solr_connection(retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print('[prerun] Giving up after 5 tries...') - sys.exit(1) - - url = os.environ.get('CKAN_SOLR_URL', '') - search_url = '{url}/select/?q=*&wt=json'.format(url=url) - http = urllib3.PoolManager() - try: - r = http.request('GET', search_url) - except urllib3.exceptions.ConnectionError as e: - print(str(e)) - print('[prerun] Unable to connect to solr, waiting...') - time.sleep(10) - check_solr_connection(retry=retry - 1) - else: - print('[prerun] Connection Status from SOLR is ', (r.status)) - -def init_db(): - - db_command = ['ckan', '-c', ckan_ini, - 'db', 'init'] - print('[prerun] Initializing or upgrading db - start') - try: - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print('[prerun] Initializing or upgrading db - end') - except subprocess.CalledProcessError as e: - if 'OperationalError' in e.output: - print(e.output) - print('[prerun] Database not ready, waiting a bit before exit...') - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - - -def init_datastore_db(): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print('[prerun] Skipping datastore initialization') - return - - datastore_perms_command = ['ckan', '-c', ckan_ini, - 'datastore', 'set-permissions'] - - connection = psycopg2.connect(conn_str) - cursor = connection.cursor() - - print('[prerun] Initializing datastore db - start') - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, - stdout=subprocess.PIPE) - - perms_sql = datastore_perms.stdout.read().decode('utf-8') - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print(notice) - - connection.commit() - - print('[prerun] Initializing datastore db - end') - print(datastore_perms.stdout.read().decode('utf-8')) - except psycopg2.Error as e: - print('[prerun] Could not initialize datastore') - print(str(e)) - - except subprocess.CalledProcessError as e: - if 'OperationalError' in e.output: - print(e.output) - print('[prerun] Database not ready, waiting a bit before exit...') - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - name = os.environ.get('CKAN_SYSADMIN_NAME') - password = os.environ.get('CKAN_SYSADMIN_PASSWORD') - email = os.environ.get('CKAN_SYSADMIN_EMAIL') - - if name and password and email: - - # Check if user exists - command = ['ckan', '-c', ckan_ini, 'user', 'show', name,] - - out = subprocess.check_output(command) - if 'User:None' not in re.sub(r'\s', '', out.decode()): - print('[prerun] Sysadmin user exists, skipping creation') - return - - # Create user - command = ['ckan', '-c', ckan_ini, 'user', 'add', - name, - 'password=' + password, - 'email=' + email] - - subprocess.call(command) - print('[prerun] Created user {0}'.format(name)) - - # Make it sysadmin - command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', - name] - - subprocess.call(command) - print('[prerun] Made user {0} a sysadmin'.format(name)) - - -if __name__ == '__main__': - - maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' - - if maintenance: - print('[prerun] Maintenance mode, skipping setup...') - else: - check_main_db_connection() - init_db() - update_plugins() - check_datastore_db_connection() - init_datastore_db() - check_solr_connection() - create_sysadmin() diff --git a/images/ckan-base/2.9/setup/start_ckan.sh b/images/ckan-base/2.9/setup/start_ckan.sh deleted file mode 100755 index 11d77c0..0000000 --- a/images/ckan-base/2.9/setup/start_ckan.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python3 prerun.py - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - -# Set the common uwsgi options -UWSGI_OPTS="--plugins http,python \ - --socket /tmp/uwsgi.sock \ - --wsgi-file /srv/app/wsgi.py \ - --module wsgi:application \ - --uid 92 --gid 92 \ - --http 0.0.0.0:5000 \ - --master --enable-threads \ - --lazy-apps \ - -p 2 -L -b 32768 --vacuum \ - --harakiri 50" - -if [ $? -eq 0 ] -then - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi - sudo -u ckan -EH uwsgi $UWSGI_OPTS -else - echo "[prerun] failed...not starting CKAN." -fi \ No newline at end of file diff --git a/images/ckan-base/2.9/setup/supervisord.conf b/images/ckan-base/2.9/setup/supervisord.conf deleted file mode 100644 index 052dbc5..0000000 --- a/images/ckan-base/2.9/setup/supervisord.conf +++ /dev/null @@ -1,23 +0,0 @@ -[unix_http_server] -file = /tmp/supervisor.sock -chmod = 0777 -chown = nobody:nogroup - -[supervisord] -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = info -pidfile = /tmp/supervisord.pid -nodaemon = true -umask = 022 -identifier = supervisor - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[include] -files = /etc/supervisord.d/*.conf \ No newline at end of file diff --git a/images/ckan-base/2.9/setup/wsgi.py b/images/ckan-base/2.9/setup/wsgi.py deleted file mode 100644 index b37d80e..0000000 --- a/images/ckan-base/2.9/setup/wsgi.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -from ckan.config.middleware import make_app -from ckan.cli import CKANConfigLoader -from logging.config import fileConfig as loggingFileConfig -config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') -abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) -loggingFileConfig(config_filepath) -config = CKANConfigLoader(config_filepath).get_config() -application = make_app(config) diff --git a/images/datapusher/Dockerfile b/images/datapusher/Dockerfile deleted file mode 100755 index 8d52845..0000000 --- a/images/datapusher/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -FROM alpine:3.13.5 - -ENV APP_DIR=/srv/app -ENV SRC_DIR=${APP_DIR}/src -ENV GIT_URL https://github.com/ckan/datapusher.git -ENV GIT_BRANCH 0.0.17 -ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py -ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars - -WORKDIR ${APP_DIR} - -RUN apk upgrade && \ - apk add --no-cache \ - python3 \ - curl \ - gcc \ - make \ - g++ \ - autoconf \ - automake \ - libtool \ - git \ - musl-dev \ - python3-dev \ - libffi-dev \ - openssl-dev \ - libxml2-dev \ - libxslt-dev \ - rust \ - cargo - -RUN apk add --no-cache \ - uwsgi \ - uwsgi-http \ - uwsgi-corerouter \ - uwsgi-python - -# Create the src directory -RUN mkdir -p ${SRC_DIR} - -# Install pip -RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ - python3 ${SRC_DIR}/get-pip.py - -# Install datapusher -RUN cd ${SRC_DIR} && \ - git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \ - cd datapusher && \ - python3 setup.py install && \ - pip3 install --no-cache-dir -r requirements.txt - -RUN cp ${APP_DIR}/src/datapusher/deployment/*.* ${APP_DIR} && \ - # Remove default values in ini file - sed -i '/http/d' ${APP_DIR}/datapusher-uwsgi.ini && \ - sed -i '/wsgi-file/d' ${APP_DIR}/datapusher-uwsgi.ini && \ - sed -i '/virtualenv/d' ${APP_DIR}/datapusher-uwsgi.ini && \ - # Remove src files - rm -rf ${APP_DIR}/src - -# Create a 'ckan' local user and group to run the app -RUN addgroup -g 92 -S www-data && \ - adduser -u 92 -h /srv/app -H -D -S -G www-data www-data - -# Set timezone -RUN echo "UTC" > /etc/timezone && \ - # Change ownership to app user - chown -R www-data:www-data /srv/app - -EXPOSE 8800 -CMD ["sh", "-c", \ - "uwsgi --plugins=http,python --http=0.0.0.0:8800 --socket=/tmp/uwsgi.sock --ini=`echo ${APP_DIR}`/datapusher-uwsgi.ini --wsgi-file=`echo ${APP_DIR}`/datapusher.wsgi"]