From 5b8a2358fa3728d1edd1aa3e8c6a9ed5752df8c3 Mon Sep 17 00:00:00 2001 From: "filip.mihajlovski" Date: Thu, 16 Feb 2023 16:42:29 +0100 Subject: [PATCH] created ckan2.10 image for alpine --- images/ckan/2.10/Dockerfile | 190 ++++++++++++++ images/ckan/2.10/Dockerfile.focal | 243 ++++++++++++++++++ images/ckan/2.10/patches/.gitkeep | 0 .../ckan/2.10/scripts/apply_ckan_patches.sh | 5 + images/ckan/2.10/setup/app/extra_scripts.sh | 4 + .../api/3/action/status_show/index.html | 25 ++ .../2.10/setup/app/maintenance/index.html | 25 ++ .../ckan/2.10/setup/app/maintenance/serve.py | 35 +++ images/ckan/2.10/setup/app/prerun.py | 230 +++++++++++++++++ images/ckan/2.10/setup/app/start_ckan.sh | 66 +++++ images/ckan/2.10/setup/app/uwsgi.conf | 2 + images/ckan/2.10/setup/app/wsgi.py | 28 ++ 12 files changed, 853 insertions(+) create mode 100644 images/ckan/2.10/Dockerfile create mode 100644 images/ckan/2.10/Dockerfile.focal create mode 100644 images/ckan/2.10/patches/.gitkeep create mode 100755 images/ckan/2.10/scripts/apply_ckan_patches.sh create mode 100755 images/ckan/2.10/setup/app/extra_scripts.sh create mode 100644 images/ckan/2.10/setup/app/maintenance/api/3/action/status_show/index.html create mode 100644 images/ckan/2.10/setup/app/maintenance/index.html create mode 100644 images/ckan/2.10/setup/app/maintenance/serve.py create mode 100644 images/ckan/2.10/setup/app/prerun.py create mode 100755 images/ckan/2.10/setup/app/start_ckan.sh create mode 100644 images/ckan/2.10/setup/app/uwsgi.conf create mode 100644 images/ckan/2.10/setup/app/wsgi.py diff --git a/images/ckan/2.10/Dockerfile b/images/ckan/2.10/Dockerfile new file mode 100644 index 0000000..becda88 --- /dev/null +++ b/images/ckan/2.10/Dockerfile @@ -0,0 +1,190 @@ +################## +### Build CKAN ### +################## +FROM alpine:3.17.2 as ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.10.0 + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.10.0 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Packages to build CKAN requirements and plugins +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl \ + postgresql-dev \ + linux-headers \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + pcre-dev \ + pcre \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +# RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 + + +########################### +### Default-Extensions #### +########################### +FROM alpine:3.17.2 as extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.2 + +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM alpine:3.17.2 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher + +# Install necessary packages to run CKAN +RUN apk add --no-cache \ + python3 \ + bash \ + git \ + gettext \ + curl \ + postgresql-client \ + libmagic \ + pcre \ + libxslt \ + libxml2 \ + tzdata \ + apache2-utils && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} + + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0 + +# Create a local user and group to run the app +RUN addgroup -g 92 -S ckan && \ + adduser -u 92 -h /srv/app -H -D -S -G ckan ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Set timezone + echo "UTC" > /etc/timezone && \ + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +ONBUILD RUN mkdir docker-afterinit.d + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] diff --git a/images/ckan/2.10/Dockerfile.focal b/images/ckan/2.10/Dockerfile.focal new file mode 100644 index 0000000..369d7fd --- /dev/null +++ b/images/ckan/2.10/Dockerfile.focal @@ -0,0 +1,243 @@ +################## +### Build CKAN ### +################## +FROM ubuntu:focal-20210827 as ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.9.7-focal + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.9.7 + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set Locale +ENV LC_ALL=en_US.UTF-8 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Set the locale +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Instal apt-utils +RUN apt-get install --no-install-recommends -y \ + apt-utils + +# Packages to build CKAN requirements and plugins +RUN apt-get install --no-install-recommends -y \ + git \ + curl \ + ca-certificates \ + python3 \ + libpq-dev \ + linux-headers-generic \ + gcc-10 \ + make \ + g++-10 \ + autoconf \ + automake \ + libtool \ + patch \ + libpcre3-dev \ + libpcre3 \ + python3-dev \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Use gcc 10 +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10 + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 + + +########################### +### Default-Extensions #### +########################### +FROM ubuntu:focal-20210827 as extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.1 + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + git \ + curl \ + ca-certificates \ + python3 \ + python3-dev + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM ubuntu:focal-20210827 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set Locale +ENV LC_ALL=en_US.UTF-8 + +# Set the locale +RUN apt-get update && \ + apt-get install --no-install-recommends -y locales && \ + sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=${LC_ALL} && \ + rm -rf /var/lib/apt/lists/* + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher + +# Install necessary packages to run CKAN +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + gettext \ + curl \ + ca-certificates \ + libpq5 \ + git \ + postgresql-client \ + python3 \ + python3-distutils \ + libpython3.8 \ + libmagic1 \ + libpcre3 \ + libxslt1.1 \ + libxml2 \ + tzdata \ + apache2-utils && \ + rm -rf /var/lib/apt/lists/* && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} && \ + # Link python to python3 + ln -s /usr/bin/python3 /usr/bin/python + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0 + +# Create a local user and group to run the app +RUN groupadd -g 92 ckan && \ + useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +ONBUILD RUN mkdir docker-afterinit.d + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] diff --git a/images/ckan/2.10/patches/.gitkeep b/images/ckan/2.10/patches/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/images/ckan/2.10/scripts/apply_ckan_patches.sh b/images/ckan/2.10/scripts/apply_ckan_patches.sh new file mode 100755 index 0000000..a7bceb9 --- /dev/null +++ b/images/ckan/2.10/scripts/apply_ckan_patches.sh @@ -0,0 +1,5 @@ +#!/bin/bash +shopt -s nullglob +for patch in patches/*.patch; do + /usr/bin/patch -p0 -i $patch +done diff --git a/images/ckan/2.10/setup/app/extra_scripts.sh b/images/ckan/2.10/setup/app/extra_scripts.sh new file mode 100755 index 0000000..80a70ef --- /dev/null +++ b/images/ckan/2.10/setup/app/extra_scripts.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# this is called before uwsgi is executed +# uset his to add extra scripts before ckan is started diff --git a/images/ckan/2.10/setup/app/maintenance/api/3/action/status_show/index.html b/images/ckan/2.10/setup/app/maintenance/api/3/action/status_show/index.html new file mode 100644 index 0000000..50276ec --- /dev/null +++ b/images/ckan/2.10/setup/app/maintenance/api/3/action/status_show/index.html @@ -0,0 +1,25 @@ + + + + + Maintenance + + +

Maintenance

+

Our data portal is currently in maintenance, please try in a while.

+ + diff --git a/images/ckan/2.10/setup/app/maintenance/index.html b/images/ckan/2.10/setup/app/maintenance/index.html new file mode 100644 index 0000000..50276ec --- /dev/null +++ b/images/ckan/2.10/setup/app/maintenance/index.html @@ -0,0 +1,25 @@ + + + + + Maintenance + + +

Maintenance

+

Our data portal is currently in maintenance, please try in a while.

+ + diff --git a/images/ckan/2.10/setup/app/maintenance/serve.py b/images/ckan/2.10/setup/app/maintenance/serve.py new file mode 100644 index 0000000..77f3bd4 --- /dev/null +++ b/images/ckan/2.10/setup/app/maintenance/serve.py @@ -0,0 +1,35 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler +import os + +PORT = 5000 + +web_dir = os.path.join(os.path.dirname(__file__)) +os.chdir(web_dir) + + +def run(server_class=ThreadingHTTPServer, handler_class=SimpleHTTPRequestHandler): + server_address = ("0.0.0.0", PORT) + httpd = server_class(server_address, handler_class) + print("Starting maintenance mode") + httpd.serve_forever() + + +if __name__ == "__main__": + run() + diff --git a/images/ckan/2.10/setup/app/prerun.py b/images/ckan/2.10/setup/app/prerun.py new file mode 100644 index 0000000..64a56bc --- /dev/null +++ b/images/ckan/2.10/setup/app/prerun.py @@ -0,0 +1,230 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import sys +import subprocess +import psycopg2 +from sqlalchemy.engine.url import make_url +import urllib.request, urllib.error, urllib.parse +import re +import json + +import time + +ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini') + +RETRY = 5 + +def check_db_connection(retry=None): + + print('[prerun] Start check_db_connection...') + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '') + try: + db_user = make_url(conn_str).username + db_passwd = make_url(conn_str).password + db_host = make_url(conn_str).host + db_name = make_url(conn_str).database + connection = psycopg2.connect(user=db_user, + host=db_host, + password=db_passwd, + database=db_name) + + except psycopg2.Error as e: + print((str(e))) + print('[prerun] Unable to connect to the database...try again in a while.') + import time + time.sleep(10) + check_db_connection(retry = retry - 1) + else: + connection.close() + +def check_solr_connection(retry=None): + + print('[prerun] Start check_solr_connection...') + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + url = os.environ.get('CKAN_SOLR_URL', '') + username = os.environ.get('SOLR_ADMIN_USERNAME', 'admin') + password = os.environ.get('SOLR_ADMIN_PASSWORD', 'pass') + search_url = '{url}/schema/name?wt=json'.format(url=url) + + try: + if not username: + connection = urllib.request.urlopen(search_url) + else: + passman = urllib.request.HTTPPasswordMgrWithDefaultRealm() + passman.add_password(None, search_url, username, password) + authhandler = urllib.request.HTTPBasicAuthHandler(passman) + opener = urllib.request.build_opener(authhandler) + urllib.request.install_opener(opener) + connection = urllib.request.urlopen(search_url) + except urllib.error.URLError as e: + print('[prerun] Unable to connect to solr...try again in a while.') + import time + time.sleep(10) + check_solr_connection(retry = retry - 1) + else: + import re + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[prerun] Succesfully connected to solr and CKAN schema loaded') + else: + print('[prerun] Succesfully connected to solr, but CKAN schema not found') + sys.exit(1) + +def init_db(): + + print('[prerun] Start init_db...') + + db_command = ['ckan', '-c', ckan_ini, 'db', 'init'] + + print('[prerun] Initializing or upgrading db - start using ckan db init') + try: + # run init scripts + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + + print('[prerun] Initializing or upgrading db - end') + except subprocess.CalledProcessError as e: + if 'OperationalError' in str(e.output): + print(e.output.decode('utf-8')) + print('[prerun] Database not ready, waiting a bit before exit...') + import time + time.sleep(5) + sys.exit(1) + else: + print(e.output.decode('utf-8')) + raise e + print('[prerun] Initializing or upgrading db - finish') + + +def init_datastore(): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print('[prerun] Skipping datastore initialization') + return + + datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore', + 'set-permissions'] + + db_user = make_url(conn_str).username + db_passwd = make_url(conn_str).password + db_host = make_url(conn_str).host + db_name = make_url(conn_str).database + connection = psycopg2.connect(user=db_user, + host=db_host, + password=db_passwd, + database=db_name) + cursor = connection.cursor() + + print('[prerun] Initializing datastore db - start') + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, + stdout=subprocess.PIPE) + + perms_sql = datastore_perms.stdout.read() + perms_sql = perms_sql.decode('utf-8') + perms_sql = perms_sql.replace("@"+db_host, "") + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print(notice) + + connection.commit() + + print('[prerun] Initializing datastore db - end') + print((datastore_perms.stdout.read())) + except psycopg2.Error as e: + print('[prerun] Could not initialize datastore') + print(e.decode('utf-8')) + + except subprocess.CalledProcessError as e: + if 'OperationalError' in str(e.output): + print(e.output.decode('utf-8')) + print('[prerun] Database not ready, waiting a bit before exit...') + time.sleep(5) + sys.exit(1) + else: + print(e.output.decode('utf-8')) + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + print('[prerun] Start create_sysadmin...') + + name = os.environ.get('CKAN_SYSADMIN_NAME') + password = os.environ.get('CKAN_SYSADMIN_PASSWORD') + email = os.environ.get('CKAN_SYSADMIN_EMAIL') + + if name and password and email: + + # Check if user exists + command = ['ckan', '-c', ckan_ini, 'user', 'show', name] + + out = subprocess.check_output(command) + if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')): + print('[prerun] Sysadmin user exists, skipping creation') + return + + # Create user + command = ['ckan', '-c', ckan_ini, 'user', 'add', + name, + 'password=' + password, + 'email=' + email] + + subprocess.call(command) + print(('[prerun] Created user {0}'.format(name))) + + # Make it sysadmin + command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', + name] + + subprocess.call(command) + print(('[prerun] Made user {0} a sysadmin'.format(name))) + +if __name__ == '__main__': + + maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' + + if maintenance: + print('[prerun] Maintenance mode, skipping setup...') + else: + check_db_connection() + check_solr_connection() + init_db() + if os.environ.get('CKAN_DATASTORE_WRITE_URL'): + init_datastore() + create_sysadmin() diff --git a/images/ckan/2.10/setup/app/start_ckan.sh b/images/ckan/2.10/setup/app/start_ckan.sh new file mode 100755 index 0000000..f1b5986 --- /dev/null +++ b/images/ckan/2.10/setup/app/start_ckan.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Run any startup scripts provided by images extending this one +if [[ -d "${APP_DIR}/docker-entrypoint.d" ]] +then + for f in ${APP_DIR}/docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini +then + echo "Setting secrets in ini file" + ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')" +fi + +echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers" +UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid ckan --gid ckan --http :5000 --master --enable-threads --wsgi-file /srv/app/wsgi.py --module wsgi:application --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch --vacuum --harakiri 50 --callable application" + +# Run the prerun script to init CKAN and create the default admin user +python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; } + +# Check if we are in maintenance mode and if yes serve the maintenance pages +if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi + +# Run any after prerun/init scripts provided by images extending this one +if [[ -d "${APP_DIR}/docker-afterinit.d" ]] +then + for f in ${APP_DIR}/docker-afterinit.d/*; do + case "$f" in + *.sh) echo "$0: Running after prerun init file $f"; . "$f" ;; + *.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully +if [ $? -eq 0 ] +then + if [ "$PASSWORD_PROTECT" = true ] + then + if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] + then + # Generate htpasswd file for basicauth + htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD + # Start uwsgi with basicauth + uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS + else + echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." + exit 1 + fi + else + # Start uwsgi + uwsgi $UWSGI_OPTS + fi +else + echo "[prerun] failed...not starting CKAN." +fi diff --git a/images/ckan/2.10/setup/app/uwsgi.conf b/images/ckan/2.10/setup/app/uwsgi.conf new file mode 100644 index 0000000..6321d6d --- /dev/null +++ b/images/ckan/2.10/setup/app/uwsgi.conf @@ -0,0 +1,2 @@ +[uwsgi] +route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd diff --git a/images/ckan/2.10/setup/app/wsgi.py b/images/ckan/2.10/setup/app/wsgi.py new file mode 100644 index 0000000..63f3a7c --- /dev/null +++ b/images/ckan/2.10/setup/app/wsgi.py @@ -0,0 +1,28 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# -*- coding: utf-8 -*- + +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join( + os.path.dirname(os.path.abspath(__file__)), u'production.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config)