Remove images directory + TODO's file

TODO's.txt

@@ -1,39 +0,0 @@
-# (from) July 7 2022
-# This repo will be actively worked on from now.
-# This file will be a ToDo/bits'n'bobs list of things to take care of
-
-ckan/ckan-docker-base: For the base images Dockerfiles (prod and dev) and related scripts
-ckan/ckan-docker: For the project-oriented image template (prod and dev). Patching only done in Dev ### This repo!
-
-All the other images should live in separate repos
-
-1. Solr -           use ckan-solr   (https://github.com/ckan/ckan-solr)
-2. PostgreSQL -     use current method (base image: postgres:12-alpine from DockerHub, enhanced in a Dockerfile) 
-                    ### This may change to be more like Solr though
-3. Redis -          use current method (DockerHub image: redis:${REDIS_VERSION} specified as a compose service in the compose file) 
-                    latest image to used is redis:6
-4. nginx -          use a late(ish) nginx image from DockerHub, enhanced in a Dockerfile)
-5. DataPusher -     built from the actual datapusher repo (https://github.com/ckan/datapusher)
-6. CKAN -           built from the ckan/ckan-base:2.9.5 base image (which is built from the ckan/ckan-docker-base repo). Include a .dev option
-7. CKAN Worker -    add new (ckan worker) container in the compose setup...maybe have it commented out initially
-
-Versions 2.9 and 2.10 (when it's out) only. Plan the repo layout for having multiple versions - OKFN could used as an example
-
-NB:  Had to update the prerun.py script as it was failing on check_solr_connection
-
-
-### ToDo (remaining things to think about and/or 'to fix') ###
-
-1.  DataPusher - needed to use a custom requirements.txt as the official didn't work (see https://github.com/ckan/datapusher/pull/251)
-2.  Use Multi-Stage images defined in Dockerfile - get the CKAN images down to bare-bones...for security reasons really
-    General Dockerfile flow of stages:
-    - base: all prod dependencies, no code yet
-    - dev, from base: all dev dependencies, no code yet (in dev, source code is bind-mounted anyway)
-    - source, from base: add code
-    - test/audit, from source: then COPY --from=dev for dev dependencies, then run tests. Optionally, audit and lint code (if you don't do it on git push already).
-    - prod, from source: no change from source stage, but listed last so in case a stage isn't targeted, the builder will default to this stage 
-    Also check out https://github.com/ckan/ckan/pull/4635 for Francesco's test stuff
-3.  CKAN Worker (maybe)
-4.  Implement SSL for the nginx container or include a howto in the docs ### NB: This has been done! albeit with a self-signed cert 
-
-

images/README

@@ -1,13 +0,0 @@
-Build the images from this directory using:
-    cd ckan-base/
-    docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile .
-    docker build -t ckan/ckan-base:testing-only.2.8 -f 2.8/Dockerfile .
-    docker build -t ckan/ckan-base:testing-only.2.7 -f 2.7/Dockerfile .
-
-Login to Dockerhub
-docker login --username=kowhai
-(will then ask for a password)
-
-Push image to (ckan) Dockerhub
-docker push ckan/ckan-base:testing-only.2.9
-<same with 2.8 and 2.7>

images/ckan-base/2.9/Dockerfile

@@ -1,103 +0,0 @@
-FROM alpine:3.13
-
-# Internal environment variables
-ENV APP_DIR=/srv/app
-ENV SRC_DIR=/srv/app/src
-ENV CKAN_INI=${APP_DIR}/ckan.ini
-ENV PIP_SRC=${SRC_DIR}
-ENV CKAN_STORAGE_PATH=/var/lib/ckan
-ENV GIT_URL=https://github.com/ckan/ckan.git
-# CKAN version to build
-ENV GIT_BRANCH=ckan-2.9.5
-# Customize these on the .env file if needed
-ENV CKAN_SITE_URL=http://localhost:5000
-ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
-
-WORKDIR ${APP_DIR}
-
-# Install necessary packages to run CKAN
-RUN apk add --no-cache tzdata \
-        git \
-        gettext \
-        postgresql-client \
-        python3 \
-        apache2-utils \
-        libxml2 \
-        libxslt \
-        musl-dev \
-        uwsgi-http \
-        uwsgi-corerouter \
-        uwsgi-python3 \
-        py3-gevent \
-        uwsgi-gevent \
-        libmagic \
-        curl \
-        sudo && \
-    # Packages to build CKAN requirements and plugins
-    apk add --no-cache --virtual .build-deps \
-        postgresql-dev \
-        gcc \
-        make \
-        g++ \
-        autoconf \
-        automake \
-	    libtool \
-        python3-dev \
-        py3-virtualenv \
-        libxml2-dev \
-        libxslt-dev \
-        linux-headers
-
-# Create SRC_DIR and install pip
-RUN mkdir -p ${SRC_DIR} && \
-    # Install pip
-    curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
-    python3 ${SRC_DIR}/get-pip.py && \     
-    rm -rf ${SRC_DIR}/get-pip.py
-
-# Set up Python3 virtual environment
-RUN cd ${APP_DIR} && \
-    python3 -m venv ${APP_DIR} && \
-    source ${APP_DIR}/bin/activate
-
-# Virtual environment binaries/scripts to be used first
-ENV PATH=${APP_DIR}/bin:${PATH}   
-    
-# Install CKAN, uwsgi, ckanext-envvars and generate CKAN config file
-RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ 
-    pip3 install uwsgi && \
-    cd ${SRC_DIR}/ckan && \
-    cp who.ini ${APP_DIR} && \
-    pip3 install -r requirement-setuptools.txt && \
-    pip3 install --no-binary :all: -r requirements.txt && \
-    # Install CKAN envvars to support loading config from environment variables
-    pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
-    # Create and update CKAN config
-    ckan generate config ${CKAN_INI}
-
-# Install and configure supervisor
-RUN pip3 install supervisor && \
-mkdir /etc/supervisord.d
-
-# Copy setup files
-COPY 2.9/setup ${APP_DIR}
-RUN mv ${APP_DIR}/supervisord.conf /etc/supervisord.conf
-
-# Create a local user and group to run the app
-RUN addgroup -g 92 -S ckan && \
-    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
-
-# Create local storage folder
-RUN mkdir -p $CKAN_STORAGE_PATH && \
-    chown -R ckan:ckan $CKAN_STORAGE_PATH
-
-# Create entrypoint directory for children image scripts
-ONBUILD RUN mkdir /docker-entrypoint.d
-
-RUN chown ckan -R /srv/app
-
-EXPOSE 5000
-
-HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
-
-CMD ["/srv/app/start_ckan.sh"] 

images/ckan-base/2.9/setup/ckan-uwsgi.ini

@@ -1,15 +0,0 @@
-[uwsgi]
-http-socket     =  :5000
-uid             =  ckan
-guid            =  ckan
-plugins         =  python3
-wsgi-file       =  /srv/app/wsgi.py
-virtualenv      =  /srv/app
-module          =  wsgi:application
-master          =  true
-processes       =  5
-pidfile         =  /tmp/%n.pid
-harakiri        =  50
-max-requests    =  5000
-vacuum          =  true
-callable        =  application

images/ckan-base/2.9/setup/prerun.py

@@ -1,194 +0,0 @@
-import os
-import sys
-import subprocess
-import psycopg2
-import urllib3
-import time
-import re
-
-ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
-
-RETRY = 5
-
-def update_plugins():
-
-    plugins = os.environ.get('CKAN__PLUGINS', '')
-    print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
-    print(plugins)
-    cmd = ['ckan', 'config-tool', ckan_ini,
-           'ckan.plugins = {}'.format(plugins)]
-    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-    print('[prerun] Plugins set.')
-
-    
-def check_main_db_connection(retry=None):
-
-    conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
-    if not conn_str:
-        print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
-    return check_db_connection(conn_str, retry)
-
-
-def check_datastore_db_connection(retry=None):
-
-    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
-    if not conn_str:
-        print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
-    return check_db_connection(conn_str, retry)
-
-
-def check_db_connection(conn_str, retry=None):
-
-    if retry is None:
-        retry = RETRY
-    elif retry == 0:
-        print('[prerun] Giving up after 5 tries...')
-        sys.exit(1)
-
-    try:
-        connection = psycopg2.connect(conn_str)
-
-    except psycopg2.Error as e:
-        print(str(e))
-        print('[prerun] Unable to connect to the database, waiting...')
-        time.sleep(10)
-        check_db_connection(conn_str, retry=retry - 1)
-    else:
-        connection.close()
-
-        
-def check_solr_connection(retry=None):
-
-    if retry is None:
-        retry = RETRY
-    elif retry == 0:
-        print('[prerun] Giving up after 5 tries...')
-        sys.exit(1)
-
-    url = os.environ.get('CKAN_SOLR_URL', '')
-    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
-    http = urllib3.PoolManager()
-    try:
-        r = http.request('GET', search_url)
-    except urllib3.exceptions.ConnectionError as e:
-        print(str(e))
-        print('[prerun] Unable to connect to solr, waiting...')
-        time.sleep(10)
-        check_solr_connection(retry=retry - 1)
-    else:
-        print('[prerun] Connection Status from SOLR is ', (r.status))
-
-def init_db():
-
-    db_command = ['ckan', '-c', ckan_ini, 
-                  'db', 'init']
-    print('[prerun] Initializing or upgrading db - start')
-    try:
-        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
-        print('[prerun] Initializing or upgrading db - end')
-    except subprocess.CalledProcessError as e:
-        if 'OperationalError' in e.output:
-            print(e.output)
-            print('[prerun] Database not ready, waiting a bit before exit...')
-            time.sleep(5)
-            sys.exit(1)
-        else:
-            print(e.output)
-            raise e
-
-
-def init_datastore_db():
-
-    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
-    if not conn_str:
-        print('[prerun] Skipping datastore initialization')
-        return
-
-    datastore_perms_command = ['ckan', '-c', ckan_ini, 
-                               'datastore', 'set-permissions']
-
-    connection = psycopg2.connect(conn_str)
-    cursor = connection.cursor()
-
-    print('[prerun] Initializing datastore db - start')
-    try:
-        datastore_perms = subprocess.Popen(
-            datastore_perms_command,
-            stdout=subprocess.PIPE)
-
-        perms_sql = datastore_perms.stdout.read().decode('utf-8')
-        # Remove internal pg command as psycopg2 does not like it
-        perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
-        cursor.execute(perms_sql)
-        for notice in connection.notices:
-            print(notice)
-
-        connection.commit()
-
-        print('[prerun] Initializing datastore db - end')
-        print(datastore_perms.stdout.read().decode('utf-8'))
-    except psycopg2.Error as e:
-        print('[prerun] Could not initialize datastore')
-        print(str(e))
-
-    except subprocess.CalledProcessError as e:
-        if 'OperationalError' in e.output:
-            print(e.output)
-            print('[prerun] Database not ready, waiting a bit before exit...')
-            time.sleep(5)
-            sys.exit(1)
-        else:
-            print(e.output)
-            raise e
-    finally:
-        cursor.close()
-        connection.close()
-
-
-def create_sysadmin():
-
-    name = os.environ.get('CKAN_SYSADMIN_NAME')
-    password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
-    email = os.environ.get('CKAN_SYSADMIN_EMAIL')
-
-    if name and password and email:
-
-        # Check if user exists
-        command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
-
-        out = subprocess.check_output(command)
-        if 'User:None' not in re.sub(r'\s', '', out.decode()):
-            print('[prerun] Sysadmin user exists, skipping creation')
-            return
-
-        # Create user
-        command = ['ckan', '-c', ckan_ini, 'user', 'add',
-                   name,
-                   'password=' + password,
-                   'email=' + email]
-
-        subprocess.call(command)
-        print('[prerun] Created user {0}'.format(name))
-
-        # Make it sysadmin
-        command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
-                   name]
-
-        subprocess.call(command)
-        print('[prerun] Made user {0} a sysadmin'.format(name))
-
-
-if __name__ == '__main__':
-
-    maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
-
-    if maintenance:
-        print('[prerun] Maintenance mode, skipping setup...')
-    else:
-        check_main_db_connection()
-        init_db()
-        update_plugins()
-        check_datastore_db_connection()
-        init_datastore_db()
-        check_solr_connection()
-        create_sysadmin()

images/ckan-base/2.9/setup/start_ckan.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-
-# Run the prerun script to init CKAN and create the default admin user
-sudo -u ckan -EH python3 prerun.py
-
-# Run any startup scripts provided by images extending this one
-if [[ -d "/docker-entrypoint.d" ]]
-then
-    for f in /docker-entrypoint.d/*; do
-        case "$f" in
-            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
-            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
-            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
-        esac
-        echo
-    done
-fi
-
-# Set the common uwsgi options
-UWSGI_OPTS="--plugins http,python \
-            --socket /tmp/uwsgi.sock \
-            --wsgi-file /srv/app/wsgi.py \
-            --module wsgi:application \
-            --uid 92 --gid 92 \
-            --http 0.0.0.0:5000 \
-            --master --enable-threads \
-            --lazy-apps \
-            -p 2 -L -b 32768 --vacuum \
-            --harakiri 50"
-
-if [ $? -eq 0 ]
-then
-    # Start supervisord
-    supervisord --configuration /etc/supervisord.conf &
-    # Start uwsgi
-    sudo -u ckan -EH uwsgi $UWSGI_OPTS
-else
-  echo "[prerun] failed...not starting CKAN."
-fi

images/ckan-base/2.9/setup/supervisord.conf

@@ -1,23 +0,0 @@
-[unix_http_server]
-file = /tmp/supervisor.sock
-chmod = 0777
-chown = nobody:nogroup
-
-[supervisord]
-logfile = /tmp/supervisord.log
-logfile_maxbytes = 50MB
-logfile_backups=10
-loglevel = info
-pidfile = /tmp/supervisord.pid
-nodaemon = true
-umask = 022
-identifier = supervisor
-
-[supervisorctl]
-serverurl = unix:///tmp/supervisor.sock
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[include]
-files = /etc/supervisord.d/*.conf

images/ckan-base/2.9/setup/wsgi.py

@@ -1,9 +0,0 @@
-import os
-from ckan.config.middleware import make_app
-from ckan.cli import CKANConfigLoader
-from logging.config import fileConfig as loggingFileConfig
-config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
-abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
-loggingFileConfig(config_filepath)
-config = CKANConfigLoader(config_filepath).get_config()
-application = make_app(config)

images/datapusher/Dockerfile

@@ -1,71 +0,0 @@
-FROM alpine:3.13.5
-
-ENV APP_DIR=/srv/app
-ENV SRC_DIR=${APP_DIR}/src
-ENV GIT_URL https://github.com/ckan/datapusher.git
-ENV GIT_BRANCH 0.0.17
-ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py
-ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
-
-WORKDIR ${APP_DIR}
-
-RUN apk upgrade && \
-    apk add --no-cache \
-        python3 \
-        curl \
-        gcc \
-        make \
-        g++ \
-        autoconf \
-        automake \
-        libtool \
-        git \
-        musl-dev \
-        python3-dev \
-        libffi-dev \
-        openssl-dev \
-        libxml2-dev \
-        libxslt-dev \
-        rust  \
-        cargo
-
-RUN apk add --no-cache \
-        uwsgi \
-        uwsgi-http \
-        uwsgi-corerouter \
-        uwsgi-python
-
-# Create the src directory
-RUN mkdir -p ${SRC_DIR}
-
-# Install pip
-RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
-    python3 ${SRC_DIR}/get-pip.py
-
-# Install datapusher
-RUN cd ${SRC_DIR} && \
-    git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \
-    cd datapusher && \
-    python3 setup.py install && \
-    pip3 install --no-cache-dir -r requirements.txt
-
-RUN cp ${APP_DIR}/src/datapusher/deployment/*.* ${APP_DIR} && \
-    # Remove default values in ini file
-    sed -i '/http/d' ${APP_DIR}/datapusher-uwsgi.ini && \
-    sed -i '/wsgi-file/d' ${APP_DIR}/datapusher-uwsgi.ini && \
-    sed -i '/virtualenv/d' ${APP_DIR}/datapusher-uwsgi.ini && \
-    # Remove src files
-    rm -rf ${APP_DIR}/src
-
-# Create a 'ckan' local user and group to run the app
-RUN addgroup -g 92 -S www-data && \
-    adduser -u 92 -h /srv/app -H -D -S -G www-data www-data
-
-# Set timezone
-RUN echo "UTC" >  /etc/timezone && \
-    # Change ownership to app user
-    chown -R www-data:www-data /srv/app
-
-EXPOSE 8800
-CMD ["sh", "-c", \
-    "uwsgi --plugins=http,python --http=0.0.0.0:8800 --socket=/tmp/uwsgi.sock --ini=`echo ${APP_DIR}`/datapusher-uwsgi.ini --wsgi-file=`echo ${APP_DIR}`/datapusher.wsgi"]