diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index ee341e5..f9d492c 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -1,11 +1,11 @@ ################## ### Build CKAN ### ################## -FROM alpine:3.8 as ckanbuild +FROM alpine:3.11 as ckanbuild # Set CKAN version to build ENV GIT_URL=https://github.com/ckan/ckan.git -ENV GIT_BRANCH=ckan-2.8.2 +ENV GIT_BRANCH=master # Set src dirs ENV SRC_DIR=/srv/app/src @@ -17,7 +17,7 @@ WORKDIR ${SRC_DIR} RUN apk add --no-cache \ git \ curl \ - python \ + python3 \ postgresql-dev \ linux-headers \ gcc \ @@ -29,7 +29,12 @@ RUN apk add --no-cache \ musl-dev \ pcre-dev \ pcre \ - python-dev + python3-dev \ + libxml2-dev \ + libxslt-dev + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python # Create the src directory RUN mkdir -p ${SRC_DIR} @@ -47,7 +52,7 @@ RUN pip wheel --wheel-dir=/wheels uwsgi gevent ############ ### MAIN ### ############ -FROM alpine:3.8 +FROM alpine:3.11 MAINTAINER Keitaro Inc @@ -65,12 +70,17 @@ RUN apk add --no-cache git \ gettext \ curl \ postgresql-client \ - python \ + python3 \ libmagic \ pcre \ + libxslt \ + libxml2 \ + tzdata \ apache2-utils && \ # Create SRC_DIR - mkdir -p ${SRC_DIR} + mkdir -p ${SRC_DIR} && \ + # Link python to python3 + ln -s /usr/bin/python3 /usr/bin/python # Install pip RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ @@ -80,9 +90,6 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ COPY --from=ckanbuild /wheels /srv/app/wheels COPY --from=ckanbuild /srv/app/src/ckan /srv/app/src/ckan -# Copy necessary scripts -COPY setup/app ${APP_DIR} - # Additional install steps for build stages artifacts RUN pip install --no-index --find-links=/srv/app/wheels uwsgi gevent @@ -98,14 +105,23 @@ RUN pip install -e /srv/app/src/ckan && \ # Install CKAN envvars to support loading config from environment variables pip install -e git+https://github.com/okfn/ckanext-envvars.git@0.0.1#egg=ckanext-envvars && \ # Create and update CKAN config - paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \ - paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # ckan generate config requires cookiecutter which is in the dev-requirements, temp workaround + pip install cookiecutter==1.6.0 && \ + # Set timezone + echo "Europe/Stockholm" > /etc/timezone && \ + ckan generate config ${APP_DIR}/production.ini && \ + # Not working atm since ckan config tool tries to load config before executing config-tool, workaround + #ckan -c ${APP_DIR}/production.ini config-tool "ckan.plugins = ${CKAN__PLUGINS}" && \ + sed -i "/ckan.plugins = stats/c ckan.plugins = ${CKAN__PLUGINS}" ${APP_DIR}/production.ini && \ # Change ownership to app user chown -R ckan:ckan /srv/app # Remove wheels RUN rm -rf /srv/app/wheels +# Copy necessary scripts +COPY setup/app ${APP_DIR} + # Create entrypoint directory for children image scripts ONBUILD RUN mkdir docker-entrypoint.d diff --git a/rootfs/setup/app/prerun.py b/rootfs/setup/app/prerun.py index 2b4eb62..5f55f70 100644 --- a/rootfs/setup/app/prerun.py +++ b/rootfs/setup/app/prerun.py @@ -2,7 +2,7 @@ import os import sys import subprocess import psycopg2 -import urllib2 +import urllib.request, urllib.error, urllib.parse import re import time @@ -13,12 +13,12 @@ RETRY = 5 def check_db_connection(retry=None): - print '[prerun] Start check_db_connection...' + print('[prerun] Start check_db_connection...') if retry is None: retry = RETRY elif retry == 0: - print '[prerun] Giving up after 5 tries...' + print('[prerun] Giving up after 5 tries...') sys.exit(1) conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '') @@ -26,8 +26,8 @@ def check_db_connection(retry=None): connection = psycopg2.connect(conn_str) except psycopg2.Error as e: - print str(e) - print '[prerun] Unable to connect to the database...try again in a while.' + print((str(e))) + print('[prerun] Unable to connect to the database...try again in a while.') import time time.sleep(10) check_db_connection(retry = retry - 1) @@ -36,70 +36,69 @@ def check_db_connection(retry=None): def check_solr_connection(retry=None): - print '[prerun] Start check_solr_connection...' + print('[prerun] Start check_solr_connection...') if retry is None: retry = RETRY elif retry == 0: - print '[prerun] Giving up after 5 tries...' + print('[prerun] Giving up after 5 tries...') sys.exit(1) url = os.environ.get('CKAN_SOLR_URL', '') search_url = '{url}/select/?q=*&wt=json'.format(url=url) try: - connection = urllib2.urlopen(search_url) - except urllib2.URLError as e: - print str(e) - print '[prerun] Unable to connect to solr...try again in a while.' + connection = urllib.request.urlopen(search_url) + except urllib.error.URLError as e: + print((str(e))) + print('[prerun] Unable to connect to solr...try again in a while.') import time time.sleep(10) check_solr_connection(retry = retry - 1) else: import re conn_info = connection.read() - conn_info = re.sub(r'"zkConnected":true', '"zkConnected":True', conn_info) eval(conn_info) def init_db(): - print '[prerun] Start init_db...' + print('[prerun] Start init_db...') - db_command = ['paster', '--plugin=ckan', 'db', 'init', '-c', ckan_ini] + db_command = ['ckan', '-c', ckan_ini, 'db', 'init'] - print '[prerun] Initializing or upgrading db - start using paster db init' + print('[prerun] Initializing or upgrading db - start using ckan db init') try: # run init scripts subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print '[prerun] Initializing or upgrading db - end' - except subprocess.CalledProcessError, e: + print('[prerun] Initializing or upgrading db - end') + except subprocess.CalledProcessError as e: if 'OperationalError' in e.output: - print e.output - print '[prerun] Database not ready, waiting a bit before exit...' + print((e.output)) + print('[prerun] Database not ready, waiting a bit before exit...') import time time.sleep(5) sys.exit(1) else: - print e.output + print((e.output)) raise e - print '[prerun] Initializing or upgrading db - finish' + print('[prerun] Initializing or upgrading db - finish') def init_datastore(): conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') if not conn_str: - print '[prerun] Skipping datastore initialization' + print('[prerun] Skipping datastore initialization') return - datastore_perms_command = ['paster', '--plugin=ckan', 'datastore', - 'set-permissions', '-c', ckan_ini] + datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore', + 'set-permissions'] connection = psycopg2.connect(conn_str) cursor = connection.cursor() - print '[prerun] Initializing datastore db - start' + print('[prerun] Initializing datastore db - start') try: datastore_perms = subprocess.Popen( datastore_perms_command, @@ -107,27 +106,27 @@ def init_datastore(): perms_sql = datastore_perms.stdout.read() # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) + perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql.decode('utf-8')) cursor.execute(perms_sql) for notice in connection.notices: - print notice + print(notice) connection.commit() - print '[prerun] Initializing datastore db - end' - print datastore_perms.stdout.read() + print('[prerun] Initializing datastore db - end') + print((datastore_perms.stdout.read())) except psycopg2.Error as e: - print '[prerun] Could not initialize datastore' - print str(e) + print('[prerun] Could not initialize datastore') + print((str(e))) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: if 'OperationalError' in e.output: - print e.output - print '[prerun] Database not ready, waiting a bit before exit...' + print((e.output)) + print('[prerun] Database not ready, waiting a bit before exit...') time.sleep(5) sys.exit(1) else: - print e.output + print((e.output)) raise e finally: cursor.close() @@ -136,7 +135,7 @@ def init_datastore(): def create_sysadmin(): - print '[prerun] Start create_sysadmin...' + print('[prerun] Start create_sysadmin...') name = os.environ.get('CKAN_SYSADMIN_NAME') password = os.environ.get('CKAN_SYSADMIN_PASSWORD') @@ -145,37 +144,35 @@ def create_sysadmin(): if name and password and email: # Check if user exists - command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini] + command = ['ckan', '-c', ckan_ini, 'user', 'show', name] out = subprocess.check_output(command) - if 'User:None' not in re.sub(r'\s', '', out): - print '[prerun] Sysadmin user exists, skipping creation' + if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')): + print('[prerun] Sysadmin user exists, skipping creation') return # Create user - command = ['paster', '--plugin=ckan', 'user', 'add', + command = ['ckan', '-c', ckan_ini, 'user', 'add', name, 'password=' + password, - 'email=' + email, - '-c', ckan_ini] + 'email=' + email] subprocess.call(command) - print '[prerun] Created user {0}'.format(name) + print(('[prerun] Created user {0}'.format(name))) # Make it sysadmin - command = ['paster', '--plugin=ckan', 'sysadmin', 'add', - name, - '-c', ckan_ini] + command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', + name] subprocess.call(command) - print '[prerun] Made user {0} a sysadmin'.format(name) + print(('[prerun] Made user {0} a sysadmin'.format(name))) if __name__ == '__main__': maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' if maintenance: - print '[prerun] Maintenance mode, skipping setup...' + print('[prerun] Maintenance mode, skipping setup...') else: check_db_connection() check_solr_connection()