Upgrade to python3 to support up and coming CKAN 2.9 version
This commit is contained in:
Marko Bocevski
parent
b037fe0199
commit
1bf976578b
|
|
@ -1,11 +1,11 @@
|
|||
##################
|
||||
### Build CKAN ###
|
||||
##################
|
||||
FROM alpine:3.8 as ckanbuild
|
||||
FROM alpine:3.11 as ckanbuild
|
||||
|
||||
# Set CKAN version to build
|
||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
||||
ENV GIT_BRANCH=ckan-2.8.2
|
||||
ENV GIT_BRANCH=master
|
||||
|
||||
# Set src dirs
|
||||
ENV SRC_DIR=/srv/app/src
|
||||
|
|
@ -17,7 +17,7 @@ WORKDIR ${SRC_DIR}
|
|||
RUN apk add --no-cache \
|
||||
git \
|
||||
curl \
|
||||
python \
|
||||
python3 \
|
||||
postgresql-dev \
|
||||
linux-headers \
|
||||
gcc \
|
||||
|
|
@ -29,7 +29,12 @@ RUN apk add --no-cache \
|
|||
musl-dev \
|
||||
pcre-dev \
|
||||
pcre \
|
||||
python-dev
|
||||
python3-dev \
|
||||
libxml2-dev \
|
||||
libxslt-dev
|
||||
|
||||
# Link python to python3
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Create the src directory
|
||||
RUN mkdir -p ${SRC_DIR}
|
||||
|
|
@ -47,7 +52,7 @@ RUN pip wheel --wheel-dir=/wheels uwsgi gevent
|
|||
############
|
||||
### MAIN ###
|
||||
############
|
||||
FROM alpine:3.8
|
||||
FROM alpine:3.11
|
||||
|
||||
MAINTAINER Keitaro Inc <info@keitaro.com>
|
||||
|
||||
|
|
@ -65,12 +70,17 @@ RUN apk add --no-cache git \
|
|||
gettext \
|
||||
curl \
|
||||
postgresql-client \
|
||||
python \
|
||||
python3 \
|
||||
libmagic \
|
||||
pcre \
|
||||
libxslt \
|
||||
libxml2 \
|
||||
tzdata \
|
||||
apache2-utils && \
|
||||
# Create SRC_DIR
|
||||
mkdir -p ${SRC_DIR}
|
||||
mkdir -p ${SRC_DIR} && \
|
||||
# Link python to python3
|
||||
ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Install pip
|
||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
||||
|
|
@ -80,9 +90,6 @@ RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
|||
COPY --from=ckanbuild /wheels /srv/app/wheels
|
||||
COPY --from=ckanbuild /srv/app/src/ckan /srv/app/src/ckan
|
||||
|
||||
# Copy necessary scripts
|
||||
COPY setup/app ${APP_DIR}
|
||||
|
||||
# Additional install steps for build stages artifacts
|
||||
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi gevent
|
||||
|
||||
|
|
@ -98,14 +105,23 @@ RUN pip install -e /srv/app/src/ckan && \
|
|||
# Install CKAN envvars to support loading config from environment variables
|
||||
pip install -e git+https://github.com/okfn/ckanext-envvars.git@0.0.1#egg=ckanext-envvars && \
|
||||
# Create and update CKAN config
|
||||
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
|
||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
||||
# ckan generate config requires cookiecutter which is in the dev-requirements, temp workaround
|
||||
pip install cookiecutter==1.6.0 && \
|
||||
# Set timezone
|
||||
echo "Europe/Stockholm" > /etc/timezone && \
|
||||
ckan generate config ${APP_DIR}/production.ini && \
|
||||
# Not working atm since ckan config tool tries to load config before executing config-tool, workaround
|
||||
#ckan -c ${APP_DIR}/production.ini config-tool "ckan.plugins = ${CKAN__PLUGINS}" && \
|
||||
sed -i "/ckan.plugins = stats/c ckan.plugins = ${CKAN__PLUGINS}" ${APP_DIR}/production.ini && \
|
||||
# Change ownership to app user
|
||||
chown -R ckan:ckan /srv/app
|
||||
|
||||
# Remove wheels
|
||||
RUN rm -rf /srv/app/wheels
|
||||
|
||||
# Copy necessary scripts
|
||||
COPY setup/app ${APP_DIR}
|
||||
|
||||
# Create entrypoint directory for children image scripts
|
||||
ONBUILD RUN mkdir docker-entrypoint.d
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import os
|
|||
import sys
|
||||
import subprocess
|
||||
import psycopg2
|
||||
import urllib2
|
||||
import urllib.request, urllib.error, urllib.parse
|
||||
import re
|
||||
|
||||
import time
|
||||
|
|
@ -13,12 +13,12 @@ RETRY = 5
|
|||
|
||||
def check_db_connection(retry=None):
|
||||
|
||||
print '[prerun] Start check_db_connection...'
|
||||
print('[prerun] Start check_db_connection...')
|
||||
|
||||
if retry is None:
|
||||
retry = RETRY
|
||||
elif retry == 0:
|
||||
print '[prerun] Giving up after 5 tries...'
|
||||
print('[prerun] Giving up after 5 tries...')
|
||||
sys.exit(1)
|
||||
|
||||
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
|
||||
|
|
@ -26,8 +26,8 @@ def check_db_connection(retry=None):
|
|||
connection = psycopg2.connect(conn_str)
|
||||
|
||||
except psycopg2.Error as e:
|
||||
print str(e)
|
||||
print '[prerun] Unable to connect to the database...try again in a while.'
|
||||
print((str(e)))
|
||||
print('[prerun] Unable to connect to the database...try again in a while.')
|
||||
import time
|
||||
time.sleep(10)
|
||||
check_db_connection(retry = retry - 1)
|
||||
|
|
@ -36,70 +36,69 @@ def check_db_connection(retry=None):
|
|||
|
||||
def check_solr_connection(retry=None):
|
||||
|
||||
print '[prerun] Start check_solr_connection...'
|
||||
print('[prerun] Start check_solr_connection...')
|
||||
|
||||
if retry is None:
|
||||
retry = RETRY
|
||||
elif retry == 0:
|
||||
print '[prerun] Giving up after 5 tries...'
|
||||
print('[prerun] Giving up after 5 tries...')
|
||||
sys.exit(1)
|
||||
|
||||
url = os.environ.get('CKAN_SOLR_URL', '')
|
||||
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
||||
|
||||
try:
|
||||
connection = urllib2.urlopen(search_url)
|
||||
except urllib2.URLError as e:
|
||||
print str(e)
|
||||
print '[prerun] Unable to connect to solr...try again in a while.'
|
||||
connection = urllib.request.urlopen(search_url)
|
||||
except urllib.error.URLError as e:
|
||||
print((str(e)))
|
||||
print('[prerun] Unable to connect to solr...try again in a while.')
|
||||
import time
|
||||
time.sleep(10)
|
||||
check_solr_connection(retry = retry - 1)
|
||||
else:
|
||||
import re
|
||||
conn_info = connection.read()
|
||||
conn_info = re.sub(r'"zkConnected":true', '"zkConnected":True', conn_info)
|
||||
eval(conn_info)
|
||||
|
||||
def init_db():
|
||||
|
||||
print '[prerun] Start init_db...'
|
||||
print('[prerun] Start init_db...')
|
||||
|
||||
db_command = ['paster', '--plugin=ckan', 'db', 'init', '-c', ckan_ini]
|
||||
db_command = ['ckan', '-c', ckan_ini, 'db', 'init']
|
||||
|
||||
print '[prerun] Initializing or upgrading db - start using paster db init'
|
||||
print('[prerun] Initializing or upgrading db - start using ckan db init')
|
||||
try:
|
||||
# run init scripts
|
||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
||||
|
||||
print '[prerun] Initializing or upgrading db - end'
|
||||
except subprocess.CalledProcessError, e:
|
||||
print('[prerun] Initializing or upgrading db - end')
|
||||
except subprocess.CalledProcessError as e:
|
||||
if 'OperationalError' in e.output:
|
||||
print e.output
|
||||
print '[prerun] Database not ready, waiting a bit before exit...'
|
||||
print((e.output))
|
||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
||||
import time
|
||||
time.sleep(5)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print e.output
|
||||
print((e.output))
|
||||
raise e
|
||||
print '[prerun] Initializing or upgrading db - finish'
|
||||
print('[prerun] Initializing or upgrading db - finish')
|
||||
|
||||
|
||||
def init_datastore():
|
||||
|
||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
||||
if not conn_str:
|
||||
print '[prerun] Skipping datastore initialization'
|
||||
print('[prerun] Skipping datastore initialization')
|
||||
return
|
||||
|
||||
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
|
||||
'set-permissions', '-c', ckan_ini]
|
||||
datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore',
|
||||
'set-permissions']
|
||||
|
||||
connection = psycopg2.connect(conn_str)
|
||||
cursor = connection.cursor()
|
||||
|
||||
print '[prerun] Initializing datastore db - start'
|
||||
print('[prerun] Initializing datastore db - start')
|
||||
try:
|
||||
datastore_perms = subprocess.Popen(
|
||||
datastore_perms_command,
|
||||
|
|
@ -107,27 +106,27 @@ def init_datastore():
|
|||
|
||||
perms_sql = datastore_perms.stdout.read()
|
||||
# Remove internal pg command as psycopg2 does not like it
|
||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql.decode('utf-8'))
|
||||
cursor.execute(perms_sql)
|
||||
for notice in connection.notices:
|
||||
print notice
|
||||
print(notice)
|
||||
|
||||
connection.commit()
|
||||
|
||||
print '[prerun] Initializing datastore db - end'
|
||||
print datastore_perms.stdout.read()
|
||||
print('[prerun] Initializing datastore db - end')
|
||||
print((datastore_perms.stdout.read()))
|
||||
except psycopg2.Error as e:
|
||||
print '[prerun] Could not initialize datastore'
|
||||
print str(e)
|
||||
print('[prerun] Could not initialize datastore')
|
||||
print((str(e)))
|
||||
|
||||
except subprocess.CalledProcessError, e:
|
||||
except subprocess.CalledProcessError as e:
|
||||
if 'OperationalError' in e.output:
|
||||
print e.output
|
||||
print '[prerun] Database not ready, waiting a bit before exit...'
|
||||
print((e.output))
|
||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
||||
time.sleep(5)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print e.output
|
||||
print((e.output))
|
||||
raise e
|
||||
finally:
|
||||
cursor.close()
|
||||
|
|
@ -136,7 +135,7 @@ def init_datastore():
|
|||
|
||||
def create_sysadmin():
|
||||
|
||||
print '[prerun] Start create_sysadmin...'
|
||||
print('[prerun] Start create_sysadmin...')
|
||||
|
||||
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
||||
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
||||
|
|
@ -145,37 +144,35 @@ def create_sysadmin():
|
|||
if name and password and email:
|
||||
|
||||
# Check if user exists
|
||||
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
|
||||
command = ['ckan', '-c', ckan_ini, 'user', 'show', name]
|
||||
|
||||
out = subprocess.check_output(command)
|
||||
if 'User:None' not in re.sub(r'\s', '', out):
|
||||
print '[prerun] Sysadmin user exists, skipping creation'
|
||||
if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')):
|
||||
print('[prerun] Sysadmin user exists, skipping creation')
|
||||
return
|
||||
|
||||
# Create user
|
||||
command = ['paster', '--plugin=ckan', 'user', 'add',
|
||||
command = ['ckan', '-c', ckan_ini, 'user', 'add',
|
||||
name,
|
||||
'password=' + password,
|
||||
'email=' + email,
|
||||
'-c', ckan_ini]
|
||||
'email=' + email]
|
||||
|
||||
subprocess.call(command)
|
||||
print '[prerun] Created user {0}'.format(name)
|
||||
print(('[prerun] Created user {0}'.format(name)))
|
||||
|
||||
# Make it sysadmin
|
||||
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
|
||||
name,
|
||||
'-c', ckan_ini]
|
||||
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
|
||||
name]
|
||||
|
||||
subprocess.call(command)
|
||||
print '[prerun] Made user {0} a sysadmin'.format(name)
|
||||
print(('[prerun] Made user {0} a sysadmin'.format(name)))
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
||||
|
||||
if maintenance:
|
||||
print '[prerun] Maintenance mode, skipping setup...'
|
||||
print('[prerun] Maintenance mode, skipping setup...')
|
||||
else:
|
||||
check_db_connection()
|
||||
check_solr_connection()
|
||||
|
|
|
|||
Loading…
Reference in New Issue