Merge pull request #27 from ckan/revert-26-new-changes-(29-Oct-2021)
Revert "New changes (29 Oct 2021)"
This commit is contained in:
Brett
GitHub
commit
359eb72f92
58
.env
58
.env
|
|
@ -1,10 +1,4 @@
|
||||||
# CKAN
|
# CKAN databases
|
||||||
CKAN_VERSION=2.9.3
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_PORT=5000
|
|
||||||
|
|
||||||
# Database
|
|
||||||
POSTGRES_USER=ckan
|
POSTGRES_USER=ckan
|
||||||
POSTGRES_PASSWORD=ckan
|
POSTGRES_PASSWORD=ckan
|
||||||
DATASTORE_READONLY_USER=datastore_ro
|
DATASTORE_READONLY_USER=datastore_ro
|
||||||
|
|
@ -14,21 +8,39 @@ CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan
|
||||||
CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
|
CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
|
||||||
CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
|
CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
|
||||||
|
|
||||||
# solr
|
# Test database connections
|
||||||
SOLR_VERSION=6.6.6
|
TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
|
||||||
CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
|
||||||
|
TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
|
||||||
|
|
||||||
# Datapusher
|
# CKAN core
|
||||||
DATAPUSHER_VERSION=0.0.17
|
CKAN_SITE_ID=default
|
||||||
CKAN_DATAPUSHER_URL=http://datapusher:8800
|
CKAN_SITE_URL=http://ckan:5000
|
||||||
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
|
CKAN_PORT=5000
|
||||||
DATAPUSHER_REWRITE_RESOURCES=True
|
CKAN_SYSADMIN_NAME=ckan_admin
|
||||||
DATAPUSHER_REWRITE_URL=http://ckan:5000
|
|
||||||
|
|
||||||
# Redis
|
|
||||||
REDIS_VERSION=6.0.7
|
|
||||||
CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
|
|
||||||
CKAN_SYSADMIN_NAME=admin
|
|
||||||
CKAN_SYSADMIN_PASSWORD=test1234
|
CKAN_SYSADMIN_PASSWORD=test1234
|
||||||
CKAN_SYSADMIN_EMAIL=admin@ckan.org
|
CKAN_SYSADMIN_EMAIL=your_email@example.com
|
||||||
|
CKAN_STORAGE_PATH=/var/lib/ckan
|
||||||
|
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
|
||||||
|
CKAN_SMTP_STARTTLS=True
|
||||||
|
CKAN_SMTP_USER=user
|
||||||
|
CKAN_SMTP_PASSWORD=pass
|
||||||
|
CKAN_SMTP_MAIL_FROM=ckan@localhost
|
||||||
|
TZ=UTC
|
||||||
|
|
||||||
|
# Other services
|
||||||
|
CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
||||||
|
CKAN_REDIS_URL=redis://redis:6379/1
|
||||||
|
CKAN_DATAPUSHER_URL=http://datapusher:8801
|
||||||
|
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
|
||||||
|
|
||||||
|
# test connections
|
||||||
|
TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
||||||
|
TEST_CKAN_REDIS_URL=redis://redis:6379/1
|
||||||
|
|
||||||
|
# Extensions
|
||||||
|
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
|
||||||
|
CKAN__HARVEST__MQ__TYPE=redis
|
||||||
|
CKAN__HARVEST__MQ__HOSTNAME=redis
|
||||||
|
CKAN__HARVEST__MQ__PORT=6379
|
||||||
|
CKAN__HARVEST__MQ__REDIS_DB=1
|
||||||
35
Dockerfile
35
Dockerfile
|
|
@ -1,35 +0,0 @@
|
||||||
FROM solr:6.6.6
|
|
||||||
|
|
||||||
# Enviroment variables
|
|
||||||
ENV SOLR_CORE ckan
|
|
||||||
|
|
||||||
# Build Arguments
|
|
||||||
ARG CKAN_VERSION
|
|
||||||
ARG SOLR_VERSION
|
|
||||||
|
|
||||||
# root user for initial config
|
|
||||||
USER root
|
|
||||||
|
|
||||||
# Create directories
|
|
||||||
RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \
|
|
||||||
mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data && \
|
|
||||||
mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index
|
|
||||||
|
|
||||||
# Add files
|
|
||||||
COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml
|
|
||||||
ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
|
|
||||||
/opt/solr/server/solr/${SOLR_CORE}/conf/
|
|
||||||
|
|
||||||
# Create core.properties
|
|
||||||
RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties
|
|
||||||
|
|
||||||
# Giving ownership to Solr
|
|
||||||
RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE}
|
|
||||||
|
|
||||||
# non-root user for runtime
|
|
||||||
USER ${SOLR_USER}:${SOLR_USER}
|
|
||||||
|
|
@ -1,37 +0,0 @@
|
||||||
FROM ckan/ckan-dev:testing-only.2.9
|
|
||||||
|
|
||||||
LABEL maintainer="brett@kowh.ai"
|
|
||||||
|
|
||||||
# Set up environment variables
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV TZ=UTC
|
|
||||||
RUN echo ${TZ} > /etc/timezone
|
|
||||||
|
|
||||||
# Make sure both files are not exactly the same
|
|
||||||
RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
|
|
||||||
cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
|
|
||||||
fi ;
|
|
||||||
|
|
||||||
# Install any extensions needed by your CKAN instance
|
|
||||||
# (Make sure to add the plugins to CKAN__PLUGINS in the .env file)
|
|
||||||
# For instance:
|
|
||||||
#RUN pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages && \
|
|
||||||
# pip install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \
|
|
||||||
# pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt
|
|
||||||
|
|
||||||
# Clone the extension(s) your are writing for your own project in the `src` folder
|
|
||||||
# to get them mounted in this image at runtime
|
|
||||||
|
|
||||||
# Apply any patches needed to CKAN core or any of the built extensions (not the
|
|
||||||
# runtime mounted ones)
|
|
||||||
# See https://github.com/okfn/docker-ckan#applying-patches
|
|
||||||
|
|
||||||
COPY patches ${APP_DIR}/patches
|
|
||||||
|
|
||||||
RUN for d in ${APP_DIR}/patches/*; do \
|
|
||||||
if [ -d $d ]; then \
|
|
||||||
for f in `ls $d/*.patch | sort -g`; do \
|
|
||||||
cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
|
|
||||||
done ; \
|
|
||||||
fi ; \
|
|
||||||
done
|
|
||||||
12
README
12
README
|
|
@ -1,12 +0,0 @@
|
||||||
Build ckan-base image (2.9)
|
|
||||||
cd /path/to/ckan-docker/ckan/ckan-base
|
|
||||||
docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile .
|
|
||||||
|
|
||||||
Login to Dockerhub
|
|
||||||
docker login --username=kowhai
|
|
||||||
(will then ask for a password)
|
|
||||||
|
|
||||||
Push image to (ckan) Dockerhub
|
|
||||||
docker push ckan/ckan-base:testing-only.2.9
|
|
||||||
|
|
||||||
Do the same with 2.8 and 2.7
|
|
||||||
|
|
@ -1,101 +0,0 @@
|
||||||
FROM alpine:3.13
|
|
||||||
|
|
||||||
# Internals, you probably don't need to change these
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_INI=${APP_DIR}/production.ini
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_STORAGE_PATH=/var/lib/ckan
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
# CKAN version to build
|
|
||||||
ENV GIT_BRANCH=ckan-2.7.11
|
|
||||||
# Customize these on the .env file if needed
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
RUN apk add --no-cache tzdata \
|
|
||||||
git \
|
|
||||||
gettext \
|
|
||||||
postgresql-client \
|
|
||||||
python \
|
|
||||||
apache2-utils \
|
|
||||||
libxml2 \
|
|
||||||
libxslt \
|
|
||||||
musl-dev \
|
|
||||||
uwsgi \
|
|
||||||
uwsgi-http \
|
|
||||||
uwsgi-corerouter \
|
|
||||||
uwsgi-python \
|
|
||||||
py2-gevent \
|
|
||||||
uwsgi-gevent \
|
|
||||||
libmagic \
|
|
||||||
curl \
|
|
||||||
sudo && \
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
apk add --no-cache --virtual .build-deps \
|
|
||||||
postgresql-dev \
|
|
||||||
gcc \
|
|
||||||
make \
|
|
||||||
g++ \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
libtool \
|
|
||||||
python-dev \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev \
|
|
||||||
linux-headers
|
|
||||||
|
|
||||||
# Create SRC_DIR
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Install pip, supervisord and uwsgi
|
|
||||||
RUN pip install supervisor && \
|
|
||||||
mkdir /etc/supervisord.d && \
|
|
||||||
#pip wheel --wheel-dir=/wheels uwsgi gevent && \
|
|
||||||
rm -rf ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
COPY 2.7/setup/supervisord.conf /etc
|
|
||||||
|
|
||||||
# Install CKAN
|
|
||||||
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
|
|
||||||
cd ${SRC_DIR}/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
# Workaround to solve https://github.com/psycopg/psycopg2/issues/594 in Alpine 3.7
|
|
||||||
sed -i -e "s/psycopg2==2.4.5/psycopg2==2.7.3.2/g" requirements.txt && \
|
|
||||||
pip install --no-binary :all: -r requirements.txt && \
|
|
||||||
# Install CKAN envvars to support loading config from environment variables
|
|
||||||
pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
|
|
||||||
# Create and update CKAN config
|
|
||||||
paster --plugin=ckan make-config ckan ${CKAN_INI} && \
|
|
||||||
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
|
|
||||||
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN addgroup -g 92 -S ckan && \
|
|
||||||
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
|
||||||
|
|
||||||
# Create local storage folder
|
|
||||||
RUN mkdir -p $CKAN_STORAGE_PATH && \
|
|
||||||
chown -R ckan:ckan $CKAN_STORAGE_PATH
|
|
||||||
|
|
||||||
COPY 2.7/setup ${APP_DIR}
|
|
||||||
COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
|
|
||||||
COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir /docker-entrypoint.d
|
|
||||||
|
|
||||||
RUN chown ckan -R /srv/app
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,197 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import subprocess
|
|
||||||
import psycopg2
|
|
||||||
import urllib2
|
|
||||||
import time
|
|
||||||
import re
|
|
||||||
|
|
||||||
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
|
|
||||||
|
|
||||||
RETRY = 5
|
|
||||||
|
|
||||||
def update_plugins():
|
|
||||||
|
|
||||||
plugins = os.environ.get('CKAN__PLUGINS', '')
|
|
||||||
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
|
|
||||||
print(plugins)
|
|
||||||
cmd = ['paster', '--plugin=ckan', 'config-tool',
|
|
||||||
ckan_ini, 'ckan.plugins = {}'.format(plugins)]
|
|
||||||
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
|
||||||
print '[prerun] Plugins set.'
|
|
||||||
|
|
||||||
|
|
||||||
def check_main_db_connection(retry=None):
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
|
|
||||||
return check_db_connection(conn_str, retry)
|
|
||||||
|
|
||||||
|
|
||||||
def check_datastore_db_connection(retry=None):
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
|
|
||||||
return check_db_connection(conn_str, retry)
|
|
||||||
|
|
||||||
|
|
||||||
def check_db_connection(conn_str, retry=None):
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print '[prerun] Giving up after 5 tries...'
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print str(e)
|
|
||||||
print '[prerun] Unable to connect to the database, waiting...'
|
|
||||||
time.sleep(10)
|
|
||||||
check_db_connection(conn_str, retry=retry - 1)
|
|
||||||
else:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def check_solr_connection(retry=None):
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print '[prerun] Giving up after 5 tries...'
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
url = os.environ.get('CKAN_SOLR_URL', '')
|
|
||||||
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection = urllib2.urlopen(search_url)
|
|
||||||
except urllib2.URLError as e:
|
|
||||||
print str(e)
|
|
||||||
print '[prerun] Unable to connect to solr, waiting...'
|
|
||||||
time.sleep(10)
|
|
||||||
check_solr_connection(retry=retry - 1)
|
|
||||||
else:
|
|
||||||
eval(connection.read())
|
|
||||||
|
|
||||||
|
|
||||||
def init_db():
|
|
||||||
|
|
||||||
db_command = ['paster', '--plugin=ckan', 'db',
|
|
||||||
'init', '-c', ckan_ini]
|
|
||||||
print '[prerun] Initializing or upgrading db - start'
|
|
||||||
try:
|
|
||||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
|
||||||
print '[prerun] Initializing or upgrading db - end'
|
|
||||||
except subprocess.CalledProcessError, e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print e.output
|
|
||||||
print '[prerun] Database not ready, waiting a bit before exit...'
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print e.output
|
|
||||||
raise e
|
|
||||||
|
|
||||||
|
|
||||||
def init_datastore_db():
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print '[prerun] Skipping datastore initialization'
|
|
||||||
return
|
|
||||||
|
|
||||||
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
|
|
||||||
'set-permissions', '-c', ckan_ini]
|
|
||||||
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
cursor = connection.cursor()
|
|
||||||
|
|
||||||
print '[prerun] Initializing datastore db - start'
|
|
||||||
try:
|
|
||||||
datastore_perms = subprocess.Popen(
|
|
||||||
datastore_perms_command,
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
perms_sql = datastore_perms.stdout.read()
|
|
||||||
# Remove internal pg command as psycopg2 does not like it
|
|
||||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
|
||||||
cursor.execute(perms_sql)
|
|
||||||
for notice in connection.notices:
|
|
||||||
print notice
|
|
||||||
|
|
||||||
connection.commit()
|
|
||||||
|
|
||||||
print '[prerun] Initializing datastore db - end'
|
|
||||||
print datastore_perms.stdout.read()
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print '[prerun] Could not initialize datastore'
|
|
||||||
print str(e)
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError, e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print e.output
|
|
||||||
print '[prerun] Database not ready, waiting a bit before exit...'
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print e.output
|
|
||||||
raise e
|
|
||||||
finally:
|
|
||||||
cursor.close()
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def create_sysadmin():
|
|
||||||
|
|
||||||
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
|
||||||
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
|
||||||
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
|
||||||
|
|
||||||
if name and password and email:
|
|
||||||
|
|
||||||
# Check if user exists
|
|
||||||
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
|
|
||||||
|
|
||||||
out = subprocess.check_output(command)
|
|
||||||
if 'User:None' not in re.sub(r'\s', '', out):
|
|
||||||
print '[prerun] Sysadmin user exists, skipping creation'
|
|
||||||
return
|
|
||||||
|
|
||||||
# Create user
|
|
||||||
command = ['paster', '--plugin=ckan', 'user', 'add',
|
|
||||||
name,
|
|
||||||
'password=' + password,
|
|
||||||
'email=' + email,
|
|
||||||
'-c', ckan_ini]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print '[prerun] Created user {0}'.format(name)
|
|
||||||
|
|
||||||
# Make it sysadmin
|
|
||||||
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
|
|
||||||
name,
|
|
||||||
'-c', ckan_ini]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print '[prerun] Made user {0} a sysadmin'.format(name)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
|
||||||
|
|
||||||
if maintenance:
|
|
||||||
print '[prerun] Maintenance mode, skipping setup...'
|
|
||||||
else:
|
|
||||||
check_main_db_connection()
|
|
||||||
init_db()
|
|
||||||
update_plugins()
|
|
||||||
check_datastore_db_connection()
|
|
||||||
init_datastore_db()
|
|
||||||
check_solr_connection()
|
|
||||||
create_sysadmin()
|
|
||||||
|
|
@ -1,47 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Run the prerun script to init CKAN and create the default admin user
|
|
||||||
sudo -u ckan -EH python prerun.py
|
|
||||||
|
|
||||||
# Run any startup scripts provided by images extending this one
|
|
||||||
if [[ -d "/docker-entrypoint.d" ]]
|
|
||||||
then
|
|
||||||
for f in /docker-entrypoint.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Set the common uwsgi options
|
|
||||||
UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
|
|
||||||
|
|
||||||
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
|
||||||
if [ $? -eq 0 ]
|
|
||||||
then
|
|
||||||
if [ "$PASSWORD_PROTECT" = true ]
|
|
||||||
then
|
|
||||||
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
|
||||||
then
|
|
||||||
# Generate htpasswd file for basicauth
|
|
||||||
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
# Start uwsgi with basicauth
|
|
||||||
sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
|
|
||||||
else
|
|
||||||
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
# Start uwsgi
|
|
||||||
sudo -u ckan -EH uwsgi $UWSGI_OPTS
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "[prerun] failed...not starting CKAN."
|
|
||||||
fi
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
[program:ckan-worker]
|
|
||||||
command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
|
|
||||||
priority=501
|
|
||||||
autostart=true
|
|
||||||
autorestart=true
|
|
||||||
redirect_stderr=true
|
|
||||||
stdout_logfile=/dev/stdout
|
|
||||||
stdout_logfile_maxbytes=0
|
|
||||||
stderr_logfile=/dev/stdout
|
|
||||||
stderr_logfile_maxbytes=0
|
|
||||||
user=ckan
|
|
||||||
environment=HOME="/srv/app",USER="ckan"
|
|
||||||
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
[unix_http_server]
|
|
||||||
file = /tmp/supervisor.sock
|
|
||||||
chmod = 0777
|
|
||||||
chown = nobody:nogroup
|
|
||||||
|
|
||||||
[supervisord]
|
|
||||||
logfile = /tmp/supervisord.log
|
|
||||||
logfile_maxbytes = 50MB
|
|
||||||
logfile_backups=10
|
|
||||||
loglevel = info
|
|
||||||
pidfile = /tmp/supervisord.pid
|
|
||||||
nodaemon = true
|
|
||||||
umask = 022
|
|
||||||
identifier = supervisor
|
|
||||||
|
|
||||||
[supervisorctl]
|
|
||||||
serverurl = unix:///tmp/supervisor.sock
|
|
||||||
|
|
||||||
[rpcinterface:supervisor]
|
|
||||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
|
||||||
|
|
||||||
[include]
|
|
||||||
files = /etc/supervisord.d/*.conf
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
FROM alpine:3.13
|
|
||||||
|
|
||||||
# Internals, you probably don't need to change these
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_INI=${APP_DIR}/production.ini
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_STORAGE_PATH=/var/lib/ckan
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
# CKAN version to build
|
|
||||||
ENV GIT_BRANCH=ckan-2.8.8
|
|
||||||
# Customize these on the .env file if needed
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
RUN apk add --no-cache tzdata \
|
|
||||||
git \
|
|
||||||
gettext \
|
|
||||||
postgresql-client \
|
|
||||||
python \
|
|
||||||
apache2-utils \
|
|
||||||
libxml2 \
|
|
||||||
libxslt \
|
|
||||||
musl-dev \
|
|
||||||
uwsgi \
|
|
||||||
uwsgi-http \
|
|
||||||
uwsgi-corerouter \
|
|
||||||
uwsgi-python \
|
|
||||||
py2-gevent \
|
|
||||||
uwsgi-gevent \
|
|
||||||
libmagic \
|
|
||||||
curl \
|
|
||||||
sudo && \
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
apk add --no-cache --virtual .build-deps \
|
|
||||||
postgresql-dev \
|
|
||||||
gcc \
|
|
||||||
make \
|
|
||||||
g++ \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
libtool \
|
|
||||||
python-dev \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev \
|
|
||||||
linux-headers
|
|
||||||
|
|
||||||
# Create SRC_DIR
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Install supervisord and uwsgi
|
|
||||||
RUN pip install supervisor && \
|
|
||||||
mkdir /etc/supervisord.d && \
|
|
||||||
#pip wheel --wheel-dir=/wheels uwsgi gevent && \
|
|
||||||
rm -rf ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
COPY 2.8/setup/supervisord.conf /etc
|
|
||||||
|
|
||||||
# Install CKAN
|
|
||||||
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
|
|
||||||
cd ${SRC_DIR}/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
pip install --no-binary :all: -r requirements.txt && \
|
|
||||||
# Install CKAN envvars to support loading config from environment variables
|
|
||||||
pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
|
|
||||||
# Create and update CKAN config
|
|
||||||
paster --plugin=ckan make-config ckan ${CKAN_INI} && \
|
|
||||||
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
|
|
||||||
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN addgroup -g 92 -S ckan && \
|
|
||||||
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
|
||||||
|
|
||||||
# Create local storage folder
|
|
||||||
RUN mkdir -p $CKAN_STORAGE_PATH && \
|
|
||||||
chown -R ckan:ckan $CKAN_STORAGE_PATH
|
|
||||||
|
|
||||||
COPY 2.8/setup ${APP_DIR}
|
|
||||||
COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
|
|
||||||
COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir /docker-entrypoint.d
|
|
||||||
|
|
||||||
RUN chown ckan -R /srv/app
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,197 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import subprocess
|
|
||||||
import psycopg2
|
|
||||||
import urllib2
|
|
||||||
import time
|
|
||||||
import re
|
|
||||||
|
|
||||||
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
|
|
||||||
|
|
||||||
RETRY = 5
|
|
||||||
|
|
||||||
def update_plugins():
|
|
||||||
|
|
||||||
plugins = os.environ.get('CKAN__PLUGINS', '')
|
|
||||||
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
|
|
||||||
print(plugins)
|
|
||||||
cmd = ['paster', '--plugin=ckan', 'config-tool',
|
|
||||||
ckan_ini, 'ckan.plugins = {}'.format(plugins)]
|
|
||||||
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
|
||||||
print '[prerun] Plugins set.'
|
|
||||||
|
|
||||||
|
|
||||||
def check_main_db_connection(retry=None):
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
|
|
||||||
return check_db_connection(conn_str, retry)
|
|
||||||
|
|
||||||
|
|
||||||
def check_datastore_db_connection(retry=None):
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
|
|
||||||
return check_db_connection(conn_str, retry)
|
|
||||||
|
|
||||||
|
|
||||||
def check_db_connection(conn_str, retry=None):
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print '[prerun] Giving up after 5 tries...'
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print str(e)
|
|
||||||
print '[prerun] Unable to connect to the database, waiting...'
|
|
||||||
time.sleep(10)
|
|
||||||
check_db_connection(conn_str, retry=retry - 1)
|
|
||||||
else:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def check_solr_connection(retry=None):
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print '[prerun] Giving up after 5 tries...'
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
url = os.environ.get('CKAN_SOLR_URL', '')
|
|
||||||
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection = urllib2.urlopen(search_url)
|
|
||||||
except urllib2.URLError as e:
|
|
||||||
print str(e)
|
|
||||||
print '[prerun] Unable to connect to solr, waiting...'
|
|
||||||
time.sleep(10)
|
|
||||||
check_solr_connection(retry=retry - 1)
|
|
||||||
else:
|
|
||||||
eval(connection.read())
|
|
||||||
|
|
||||||
|
|
||||||
def init_db():
|
|
||||||
|
|
||||||
db_command = ['paster', '--plugin=ckan', 'db',
|
|
||||||
'init', '-c', ckan_ini]
|
|
||||||
print '[prerun] Initializing or upgrading db - start'
|
|
||||||
try:
|
|
||||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
|
||||||
print '[prerun] Initializing or upgrading db - end'
|
|
||||||
except subprocess.CalledProcessError, e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print e.output
|
|
||||||
print '[prerun] Database not ready, waiting a bit before exit...'
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print e.output
|
|
||||||
raise e
|
|
||||||
|
|
||||||
|
|
||||||
def init_datastore_db():
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print '[prerun] Skipping datastore initialization'
|
|
||||||
return
|
|
||||||
|
|
||||||
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
|
|
||||||
'set-permissions', '-c', ckan_ini]
|
|
||||||
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
cursor = connection.cursor()
|
|
||||||
|
|
||||||
print '[prerun] Initializing datastore db - start'
|
|
||||||
try:
|
|
||||||
datastore_perms = subprocess.Popen(
|
|
||||||
datastore_perms_command,
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
perms_sql = datastore_perms.stdout.read()
|
|
||||||
# Remove internal pg command as psycopg2 does not like it
|
|
||||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
|
||||||
cursor.execute(perms_sql)
|
|
||||||
for notice in connection.notices:
|
|
||||||
print notice
|
|
||||||
|
|
||||||
connection.commit()
|
|
||||||
|
|
||||||
print '[prerun] Initializing datastore db - end'
|
|
||||||
print datastore_perms.stdout.read()
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print '[prerun] Could not initialize datastore'
|
|
||||||
print str(e)
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError, e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print e.output
|
|
||||||
print '[prerun] Database not ready, waiting a bit before exit...'
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print e.output
|
|
||||||
raise e
|
|
||||||
finally:
|
|
||||||
cursor.close()
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def create_sysadmin():
|
|
||||||
|
|
||||||
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
|
||||||
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
|
||||||
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
|
||||||
|
|
||||||
if name and password and email:
|
|
||||||
|
|
||||||
# Check if user exists
|
|
||||||
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
|
|
||||||
|
|
||||||
out = subprocess.check_output(command)
|
|
||||||
if 'User:None' not in re.sub(r'\s', '', out):
|
|
||||||
print '[prerun] Sysadmin user exists, skipping creation'
|
|
||||||
return
|
|
||||||
|
|
||||||
# Create user
|
|
||||||
command = ['paster', '--plugin=ckan', 'user', 'add',
|
|
||||||
name,
|
|
||||||
'password=' + password,
|
|
||||||
'email=' + email,
|
|
||||||
'-c', ckan_ini]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print '[prerun] Created user {0}'.format(name)
|
|
||||||
|
|
||||||
# Make it sysadmin
|
|
||||||
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
|
|
||||||
name,
|
|
||||||
'-c', ckan_ini]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print '[prerun] Made user {0} a sysadmin'.format(name)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
|
||||||
|
|
||||||
if maintenance:
|
|
||||||
print '[prerun] Maintenance mode, skipping setup...'
|
|
||||||
else:
|
|
||||||
check_main_db_connection()
|
|
||||||
init_db()
|
|
||||||
update_plugins()
|
|
||||||
check_datastore_db_connection()
|
|
||||||
init_datastore_db()
|
|
||||||
check_solr_connection()
|
|
||||||
create_sysadmin()
|
|
||||||
|
|
@ -1,47 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Run the prerun script to init CKAN and create the default admin user
|
|
||||||
sudo -u ckan -EH python prerun.py
|
|
||||||
|
|
||||||
# Run any startup scripts provided by images extending this one
|
|
||||||
if [[ -d "/docker-entrypoint.d" ]]
|
|
||||||
then
|
|
||||||
for f in /docker-entrypoint.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Set the common uwsgi options
|
|
||||||
UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
|
|
||||||
|
|
||||||
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
|
||||||
if [ $? -eq 0 ]
|
|
||||||
then
|
|
||||||
if [ "$PASSWORD_PROTECT" = true ]
|
|
||||||
then
|
|
||||||
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
|
||||||
then
|
|
||||||
# Generate htpasswd file for basicauth
|
|
||||||
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
# Start uwsgi with basicauth
|
|
||||||
sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
|
|
||||||
else
|
|
||||||
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
# Start uwsgi
|
|
||||||
sudo -u ckan -EH uwsgi $UWSGI_OPTS
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "[prerun] failed...not starting CKAN."
|
|
||||||
fi
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
[program:ckan-worker]
|
|
||||||
command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
|
|
||||||
priority=501
|
|
||||||
autostart=true
|
|
||||||
autorestart=true
|
|
||||||
redirect_stderr=true
|
|
||||||
stdout_logfile=/dev/stdout
|
|
||||||
stdout_logfile_maxbytes=0
|
|
||||||
stderr_logfile=/dev/stdout
|
|
||||||
stderr_logfile_maxbytes=0
|
|
||||||
user=ckan
|
|
||||||
environment=HOME="/srv/app",USER="ckan"
|
|
||||||
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
[unix_http_server]
|
|
||||||
file = /tmp/supervisor.sock
|
|
||||||
chmod = 0777
|
|
||||||
chown = nobody:nogroup
|
|
||||||
|
|
||||||
[supervisord]
|
|
||||||
logfile = /tmp/supervisord.log
|
|
||||||
logfile_maxbytes = 50MB
|
|
||||||
logfile_backups=10
|
|
||||||
loglevel = info
|
|
||||||
pidfile = /tmp/supervisord.pid
|
|
||||||
nodaemon = true
|
|
||||||
umask = 022
|
|
||||||
identifier = supervisor
|
|
||||||
|
|
||||||
[supervisorctl]
|
|
||||||
serverurl = unix:///tmp/supervisor.sock
|
|
||||||
|
|
||||||
[rpcinterface:supervisor]
|
|
||||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
|
||||||
|
|
||||||
[include]
|
|
||||||
files = /etc/supervisord.d/*.conf
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
|
||||||
|
|
@ -1,109 +0,0 @@
|
||||||
FROM alpine:3.13
|
|
||||||
|
|
||||||
# Internal environment variables
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_INI=${APP_DIR}/ckan.ini
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_STORAGE_PATH=/var/lib/ckan
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
# CKAN version to build
|
|
||||||
ENV GIT_BRANCH=ckan-2.9.4
|
|
||||||
# Customize these on the .env file if needed
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
RUN apk add --no-cache tzdata \
|
|
||||||
git \
|
|
||||||
gettext \
|
|
||||||
postgresql-client \
|
|
||||||
python3 \
|
|
||||||
apache2-utils \
|
|
||||||
libxml2 \
|
|
||||||
libxslt \
|
|
||||||
musl-dev \
|
|
||||||
uwsgi-http \
|
|
||||||
uwsgi-corerouter \
|
|
||||||
uwsgi-python3 \
|
|
||||||
py3-gevent \
|
|
||||||
uwsgi-gevent \
|
|
||||||
libmagic \
|
|
||||||
curl \
|
|
||||||
sudo && \
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
apk add --no-cache --virtual .build-deps \
|
|
||||||
postgresql-dev \
|
|
||||||
gcc \
|
|
||||||
make \
|
|
||||||
g++ \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
libtool \
|
|
||||||
python3-dev \
|
|
||||||
py3-virtualenv \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev \
|
|
||||||
linux-headers
|
|
||||||
|
|
||||||
# Create src directory (SRC_DIR)
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
|
||||||
python3 ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
# Set up Python3 virtual environment
|
|
||||||
RUN cd ${APP_DIR} && \
|
|
||||||
python3 -m venv ${APP_DIR} && \
|
|
||||||
source ${APP_DIR}/bin/activate
|
|
||||||
|
|
||||||
# Virtual environment binaries/scripts to be used first
|
|
||||||
ENV PATH=${APP_DIR}/bin:${PATH}
|
|
||||||
|
|
||||||
# Install CKAN code
|
|
||||||
RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
|
||||||
|
|
||||||
# Install uwsgi
|
|
||||||
RUN pip3 install uwsgi
|
|
||||||
|
|
||||||
# Complete CKAN install
|
|
||||||
RUN cd ${SRC_DIR}/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
pip3 install -r requirement-setuptools.txt && \
|
|
||||||
pip3 install --no-binary :all: -r requirements.txt && \
|
|
||||||
# Install CKAN envvars to support loading config from environment variables
|
|
||||||
pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars
|
|
||||||
|
|
||||||
# Create and update CKAN config
|
|
||||||
RUN ckan generate config ${CKAN_INI}
|
|
||||||
|
|
||||||
# Install and configure supervisor
|
|
||||||
RUN pip3 install supervisor && \
|
|
||||||
mkdir /etc/supervisord.d
|
|
||||||
|
|
||||||
# Copy all setup files
|
|
||||||
COPY 2.9/setup ${APP_DIR}
|
|
||||||
COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
|
|
||||||
COPY 2.9/setup/supervisord.conf /etc/supervisord.conf
|
|
||||||
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN addgroup -g 92 -S ckan && \
|
|
||||||
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
|
||||||
|
|
||||||
# Create local storage folder
|
|
||||||
RUN mkdir -p $CKAN_STORAGE_PATH && \
|
|
||||||
chown -R ckan:ckan $CKAN_STORAGE_PATH
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir /docker-entrypoint.d
|
|
||||||
|
|
||||||
RUN chown ckan -R /srv/app
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
http-socket = :5000
|
|
||||||
uid = ckan
|
|
||||||
guid = ckan
|
|
||||||
plugins = python3
|
|
||||||
wsgi-file = /srv/app/wsgi.py
|
|
||||||
virtualenv = /srv/app
|
|
||||||
module = wsgi:application
|
|
||||||
master = true
|
|
||||||
processes = 5
|
|
||||||
pidfile = /tmp/%n.pid
|
|
||||||
harakiri = 50
|
|
||||||
max-requests = 5000
|
|
||||||
vacuum = true
|
|
||||||
callable = application
|
|
||||||
|
|
@ -1,194 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import subprocess
|
|
||||||
import psycopg2
|
|
||||||
import urllib3
|
|
||||||
import time
|
|
||||||
import re
|
|
||||||
|
|
||||||
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
|
|
||||||
|
|
||||||
RETRY = 5
|
|
||||||
|
|
||||||
def update_plugins():
|
|
||||||
|
|
||||||
plugins = os.environ.get('CKAN__PLUGINS', '')
|
|
||||||
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
|
|
||||||
print(plugins)
|
|
||||||
cmd = ['ckan', 'config-tool', ckan_ini,
|
|
||||||
'ckan.plugins = {}'.format(plugins)]
|
|
||||||
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
|
||||||
print('[prerun] Plugins set.')
|
|
||||||
|
|
||||||
|
|
||||||
def check_main_db_connection(retry=None):
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
|
|
||||||
return check_db_connection(conn_str, retry)
|
|
||||||
|
|
||||||
|
|
||||||
def check_datastore_db_connection(retry=None):
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
|
|
||||||
return check_db_connection(conn_str, retry)
|
|
||||||
|
|
||||||
|
|
||||||
def check_db_connection(conn_str, retry=None):
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print(str(e))
|
|
||||||
print('[prerun] Unable to connect to the database, waiting...')
|
|
||||||
time.sleep(10)
|
|
||||||
check_db_connection(conn_str, retry=retry - 1)
|
|
||||||
else:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def check_solr_connection(retry=None):
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
url = os.environ.get('CKAN_SOLR_URL', '')
|
|
||||||
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
|
||||||
http = urllib3.PoolManager()
|
|
||||||
try:
|
|
||||||
r = http.request('GET', search_url)
|
|
||||||
except urllib3.exceptions.ConnectionError as e:
|
|
||||||
print(str(e))
|
|
||||||
print('[prerun] Unable to connect to solr, waiting...')
|
|
||||||
time.sleep(10)
|
|
||||||
check_solr_connection(retry=retry - 1)
|
|
||||||
else:
|
|
||||||
print('[prerun] Connection Status from SOLR is ', (r.status))
|
|
||||||
|
|
||||||
def init_db():
|
|
||||||
|
|
||||||
db_command = ['ckan', '-c', ckan_ini,
|
|
||||||
'db', 'init']
|
|
||||||
print('[prerun] Initializing or upgrading db - start')
|
|
||||||
try:
|
|
||||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
|
||||||
print('[prerun] Initializing or upgrading db - end')
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print(e.output)
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print(e.output)
|
|
||||||
raise e
|
|
||||||
|
|
||||||
|
|
||||||
def init_datastore_db():
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] Skipping datastore initialization')
|
|
||||||
return
|
|
||||||
|
|
||||||
datastore_perms_command = ['ckan', '-c', ckan_ini,
|
|
||||||
'datastore', 'set-permissions']
|
|
||||||
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
cursor = connection.cursor()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - start')
|
|
||||||
try:
|
|
||||||
datastore_perms = subprocess.Popen(
|
|
||||||
datastore_perms_command,
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
perms_sql = datastore_perms.stdout.read().decode('utf-8')
|
|
||||||
# Remove internal pg command as psycopg2 does not like it
|
|
||||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
|
||||||
cursor.execute(perms_sql)
|
|
||||||
for notice in connection.notices:
|
|
||||||
print(notice)
|
|
||||||
|
|
||||||
connection.commit()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - end')
|
|
||||||
print(datastore_perms.stdout.read().decode('utf-8'))
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print('[prerun] Could not initialize datastore')
|
|
||||||
print(str(e))
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print(e.output)
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print(e.output)
|
|
||||||
raise e
|
|
||||||
finally:
|
|
||||||
cursor.close()
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def create_sysadmin():
|
|
||||||
|
|
||||||
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
|
||||||
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
|
||||||
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
|
||||||
|
|
||||||
if name and password and email:
|
|
||||||
|
|
||||||
# Check if user exists
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
|
|
||||||
|
|
||||||
out = subprocess.check_output(command)
|
|
||||||
if 'User:None' not in re.sub(r'\s', '', out.decode()):
|
|
||||||
print('[prerun] Sysadmin user exists, skipping creation')
|
|
||||||
return
|
|
||||||
|
|
||||||
# Create user
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'user', 'add',
|
|
||||||
name,
|
|
||||||
'password=' + password,
|
|
||||||
'email=' + email]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print('[prerun] Created user {0}'.format(name))
|
|
||||||
|
|
||||||
# Make it sysadmin
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
|
|
||||||
name]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print('[prerun] Made user {0} a sysadmin'.format(name))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
|
||||||
|
|
||||||
if maintenance:
|
|
||||||
print('[prerun] Maintenance mode, skipping setup...')
|
|
||||||
else:
|
|
||||||
check_main_db_connection()
|
|
||||||
init_db()
|
|
||||||
update_plugins()
|
|
||||||
check_datastore_db_connection()
|
|
||||||
init_datastore_db()
|
|
||||||
check_solr_connection()
|
|
||||||
create_sysadmin()
|
|
||||||
|
|
@ -1,46 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Run the prerun script to init CKAN and create the default admin user
|
|
||||||
sudo -u ckan -EH python3 prerun.py
|
|
||||||
|
|
||||||
# Run any startup scripts provided by images extending this one
|
|
||||||
if [[ -d "/docker-entrypoint.d" ]]
|
|
||||||
then
|
|
||||||
for f in /docker-entrypoint.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
|
||||||
if [ $? -eq 0 ]
|
|
||||||
then
|
|
||||||
if [ "$PASSWORD_PROTECT" = true ]
|
|
||||||
then
|
|
||||||
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
|
||||||
then
|
|
||||||
# Generate htpasswd file for basicauth
|
|
||||||
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
# Start uwsgi with basicauth
|
|
||||||
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
|
|
||||||
else
|
|
||||||
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
# Start uwsgi
|
|
||||||
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "[prerun] failed...not starting CKAN."
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
[program:ckan-worker]
|
|
||||||
command=ckan -c /srv/app/ckan.ini jobs worker
|
|
||||||
priority=501
|
|
||||||
autostart=true
|
|
||||||
autorestart=true
|
|
||||||
redirect_stderr=true
|
|
||||||
stdout_logfile=/dev/stdout
|
|
||||||
stdout_logfile_maxbytes=0
|
|
||||||
stderr_logfile=/dev/stdout
|
|
||||||
stderr_logfile_maxbytes=0
|
|
||||||
user=ckan
|
|
||||||
environment=HOME="/srv/app",USER="ckan"
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
[unix_http_server]
|
|
||||||
file = /tmp/supervisor.sock
|
|
||||||
chmod = 0777
|
|
||||||
chown = nobody:nogroup
|
|
||||||
|
|
||||||
[supervisord]
|
|
||||||
logfile = /tmp/supervisord.log
|
|
||||||
logfile_maxbytes = 50MB
|
|
||||||
logfile_backups=10
|
|
||||||
loglevel = info
|
|
||||||
pidfile = /tmp/supervisord.pid
|
|
||||||
nodaemon = true
|
|
||||||
umask = 022
|
|
||||||
identifier = supervisor
|
|
||||||
|
|
||||||
[supervisorctl]
|
|
||||||
serverurl = unix:///tmp/supervisor.sock
|
|
||||||
|
|
||||||
[rpcinterface:supervisor]
|
|
||||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
|
||||||
|
|
||||||
[include]
|
|
||||||
files = /etc/supervisord.d/*.conf
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
import os
|
|
||||||
from ckan.config.middleware import make_app
|
|
||||||
from ckan.cli import CKANConfigLoader
|
|
||||||
from logging.config import fileConfig as loggingFileConfig
|
|
||||||
config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
|
|
||||||
abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
|
|
||||||
loggingFileConfig(config_filepath)
|
|
||||||
config = CKANConfigLoader(config_filepath).get_config()
|
|
||||||
application = make_app(config)
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
FROM ckan/ckan-base:testing-only.2.7
|
|
||||||
|
|
||||||
LABEL maintainer="brett@kowh.ai"
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
|
|
||||||
|
|
||||||
# Install packages needed by the dev requirements
|
|
||||||
RUN apk add --no-cache libffi-dev
|
|
||||||
|
|
||||||
# Install CKAN dev requirements
|
|
||||||
RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
|
|
||||||
|
|
||||||
# Create folder for local extensions sources
|
|
||||||
RUN mkdir $SRC_EXTENSIONS_DIR
|
|
||||||
|
|
||||||
COPY setup/start_ckan_development.sh ${APP_DIR}
|
|
||||||
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan_development.sh"]
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
FROM ckan/ckan-base:testing-only.2.8
|
|
||||||
|
|
||||||
LABEL maintainer="brett@kowh.ai"
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
|
|
||||||
|
|
||||||
# Install packages needed by the dev requirements
|
|
||||||
RUN apk add --no-cache libffi-dev
|
|
||||||
|
|
||||||
# Install CKAN dev requirements
|
|
||||||
RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
|
|
||||||
|
|
||||||
# Create folder for local extensions sources
|
|
||||||
RUN mkdir $SRC_EXTENSIONS_DIR
|
|
||||||
|
|
||||||
COPY setup/start_ckan_development.sh ${APP_DIR}
|
|
||||||
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan_development.sh"]
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
FROM ckan/ckan-base:testing-only.2.9
|
|
||||||
|
|
||||||
LABEL maintainer="brett@kowh.ai"
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
|
|
||||||
|
|
||||||
# Install packages needed by the dev requirements
|
|
||||||
RUN apk add --no-cache libffi-dev
|
|
||||||
|
|
||||||
# Set up Python3 virtual environment
|
|
||||||
RUN cd ${APP_DIR} && \
|
|
||||||
source ${APP_DIR}/bin/activate
|
|
||||||
|
|
||||||
# Virtual environment binaries/scripts to be used first
|
|
||||||
ENV PATH=${APP_DIR}/bin:${PATH}
|
|
||||||
|
|
||||||
# Install CKAN dev requirements
|
|
||||||
# Will need to change this eventually - when CKAN 2.9 is out
|
|
||||||
# wget https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
|
|
||||||
# RUN pip3 install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
|
|
||||||
RUN pip3 install -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
|
|
||||||
|
|
||||||
# Create folder for local extensions sources
|
|
||||||
RUN mkdir $SRC_EXTENSIONS_DIR
|
|
||||||
|
|
||||||
COPY setup/start_ckan_development.sh ${APP_DIR}
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan_development.sh"]
|
|
||||||
|
|
@ -1,82 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Install any local extensions in the src_extensions volume
|
|
||||||
echo "Looking for local extensions to install..."
|
|
||||||
echo "Extension dir contents:"
|
|
||||||
ls -la $SRC_EXTENSIONS_DIR
|
|
||||||
for i in $SRC_EXTENSIONS_DIR/*
|
|
||||||
do
|
|
||||||
if [ -d $i ];
|
|
||||||
then
|
|
||||||
|
|
||||||
if [ -f $i/pip-requirements.txt ];
|
|
||||||
then
|
|
||||||
pip install -r $i/pip-requirements.txt
|
|
||||||
echo "Found requirements file in $i"
|
|
||||||
fi
|
|
||||||
if [ -f $i/requirements.txt ];
|
|
||||||
then
|
|
||||||
pip install -r $i/requirements.txt
|
|
||||||
echo "Found requirements file in $i"
|
|
||||||
fi
|
|
||||||
if [ -f $i/dev-requirements.txt ];
|
|
||||||
then
|
|
||||||
pip install -r $i/dev-requirements.txt
|
|
||||||
echo "Found dev-requirements file in $i"
|
|
||||||
fi
|
|
||||||
if [ -f $i/setup.py ];
|
|
||||||
then
|
|
||||||
cd $i
|
|
||||||
python $i/setup.py develop
|
|
||||||
echo "Found setup.py file in $i"
|
|
||||||
cd $APP_DIR
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Point `use` in test.ini to location of `test-core.ini`
|
|
||||||
if [ -f $i/test.ini ];
|
|
||||||
then
|
|
||||||
echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i"
|
|
||||||
paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# Set debug to true
|
|
||||||
echo "Enabling debug mode"
|
|
||||||
ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
|
|
||||||
|
|
||||||
# Update the plugins setting in the ini file with the values defined in the env var
|
|
||||||
echo "Loading the following plugins: $CKAN__PLUGINS"
|
|
||||||
ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
|
|
||||||
|
|
||||||
# Update test-core.ini DB, SOLR & Redis settings
|
|
||||||
echo "Loading test settings into test-core.ini"
|
|
||||||
ckan config-tool $SRC_DIR/ckan/test-core.ini \
|
|
||||||
"sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \
|
|
||||||
"ckan.datstore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \
|
|
||||||
"ckan.datstore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \
|
|
||||||
"solr_url = $TEST_CKAN_SOLR_URL" \
|
|
||||||
"ckan.redis_url = $TEST_CKAN_REDIS_URL"
|
|
||||||
|
|
||||||
# Run the prerun script to init CKAN and create the default admin user
|
|
||||||
sudo -u ckan -EH python prerun.py
|
|
||||||
|
|
||||||
# Run any startup scripts provided by images extending this one
|
|
||||||
if [[ -d "/docker-entrypoint.d" ]]
|
|
||||||
then
|
|
||||||
for f in /docker-entrypoint.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
|
|
||||||
# Start the development server with automatic reload
|
|
||||||
# Check the --reloader options sudo -u ckan -EH ckan -c $CKAN_INI run --reloader <TEXT>
|
|
||||||
sudo -u ckan -EH ckan -c $CKAN_INI run
|
|
||||||
|
|
@ -1,10 +1,5 @@
|
||||||
version: "3"
|
version: "3"
|
||||||
|
|
||||||
volumes:
|
|
||||||
pg_data:
|
|
||||||
solr_data:
|
|
||||||
ckan_storage:
|
|
||||||
|
|
||||||
services:
|
services:
|
||||||
|
|
||||||
nginx:
|
nginx:
|
||||||
|
|
@ -36,11 +31,16 @@ services:
|
||||||
volumes:
|
volumes:
|
||||||
- ckan_storage:/var/lib/ckan
|
- ckan_storage:/var/lib/ckan
|
||||||
|
|
||||||
|
datapusher:
|
||||||
|
container_name: datapusher
|
||||||
|
image: kowhai/datapusher:0.0.17
|
||||||
|
ports:
|
||||||
|
- "8800:8800"
|
||||||
|
|
||||||
db:
|
db:
|
||||||
container_name: db
|
container_name: db
|
||||||
build:
|
build:
|
||||||
context: .
|
context: postgresql/
|
||||||
dockerfile: postgresql/Dockerfile
|
|
||||||
args:
|
args:
|
||||||
- DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
|
- DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
||||||
|
|
@ -50,31 +50,19 @@ services:
|
||||||
- PGDATA=/var/lib/postgresql/data/db
|
- PGDATA=/var/lib/postgresql/data/db
|
||||||
volumes:
|
volumes:
|
||||||
- pg_data:/var/lib/postgresql/data
|
- pg_data:/var/lib/postgresql/data
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "pg_isready", "-U", "ckan"]
|
|
||||||
|
|
||||||
datapusher:
|
|
||||||
container_name: datapusher
|
|
||||||
build:
|
|
||||||
context: datapusher
|
|
||||||
ports:
|
|
||||||
- "8800:8800"
|
|
||||||
environment:
|
|
||||||
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
|
|
||||||
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
|
|
||||||
- CKAN__DATAPUSHER__CALLBACK_URL_BASE=${CKAN__DATAPUSHER__CALLBACK_URL_BASE}
|
|
||||||
|
|
||||||
solr:
|
solr:
|
||||||
container_name: solr
|
container_name: solr
|
||||||
build:
|
build:
|
||||||
context: .
|
context: solr/
|
||||||
dockerfile: solr/Dockerfile
|
|
||||||
args:
|
|
||||||
- SOLR_VERSION=${SOLR_VERSION}
|
|
||||||
- CKAN_VERSION=${CKAN_VERSION}
|
|
||||||
volumes:
|
volumes:
|
||||||
- solr_data:/opt/solr/server/solr/ckan/data
|
- solr_data:/opt/solr/server/solr/ckan/data/index
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
container_name: redis
|
container_name: redis
|
||||||
image: redis:${REDIS_VERSION}
|
image: redis:alpine
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
ckan_storage:
|
||||||
|
pg_data:
|
||||||
|
solr_data:
|
||||||
|
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
\set datastore_ro_password '\'' `echo $DATASTORE_READONLY_PASSWORD` '\''
|
|
||||||
|
|
||||||
CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD :datastore_ro_password;
|
|
||||||
CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8';
|
|
||||||
CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8';
|
|
||||||
22
index.html
22
index.html
|
|
@ -1,22 +0,0 @@
|
||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
|
||||||
<title>CKAN Docker NGINX landing page</title>
|
|
||||||
<style>
|
|
||||||
h1{
|
|
||||||
font-weight:lighter;
|
|
||||||
font-family: Arial, Helvetica, sans-serif;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
|
|
||||||
<h2>
|
|
||||||
CKAN Docker NGINX landing page
|
|
||||||
</h2>
|
|
||||||
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
69
notes.txt
69
notes.txt
|
|
@ -1,69 +0,0 @@
|
||||||
|
|
||||||
Environment variables are in the .env file and loaded into the docker-compose.yml file at runtime.
|
|
||||||
The Dockerfile will need an ARG <value> line also
|
|
||||||
|
|
||||||
List all env variables and decide where they go
|
|
||||||
.env
|
|
||||||
env-ckan
|
|
||||||
ARG
|
|
||||||
ENV
|
|
||||||
|
|
||||||
NB: .env variables can be referenced in the docker-compose.yml file (environment: part of the service)
|
|
||||||
The docker-compose.yml file can pass these variables to the Dockerfile as ARG parameters
|
|
||||||
--------------------------- from OKFN ---------------------------
|
|
||||||
# DB image settings
|
|
||||||
POSTGRES_PASSWORD=ckan
|
|
||||||
DATASTORE_READONLY_PASSWORD=datastore
|
|
||||||
|
|
||||||
# Basic
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://ckan:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_SYSADMIN_NAME=ckan_admin
|
|
||||||
CKAN_SYSADMIN_PASSWORD=test1234
|
|
||||||
CKAN_SYSADMIN_EMAIL=your_email@example.com
|
|
||||||
TZ=UTC
|
|
||||||
|
|
||||||
# Database connections (TODO: avoid duplication)
|
|
||||||
CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan
|
|
||||||
CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
|
|
||||||
CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
|
|
||||||
|
|
||||||
# Test database connections
|
|
||||||
TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
|
|
||||||
TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
|
|
||||||
TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
|
|
||||||
|
|
||||||
# Other services connections
|
|
||||||
CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
CKAN_DATAPUSHER_URL=http://datapusher:8800
|
|
||||||
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
|
|
||||||
|
|
||||||
TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
TEST_CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
|
|
||||||
# Core settings
|
|
||||||
CKAN__STORAGE_PATH=/var/lib/ckan
|
|
||||||
|
|
||||||
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
|
|
||||||
CKAN_SMTP_STARTTLS=True
|
|
||||||
CKAN_SMTP_USER=user
|
|
||||||
CKAN_SMTP_PASSWORD=pass
|
|
||||||
CKAN_SMTP_MAIL_FROM=ckan@localhost
|
|
||||||
|
|
||||||
# Extensions
|
|
||||||
|
|
||||||
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
|
|
||||||
CKAN__HARVEST__MQ__TYPE=redis
|
|
||||||
CKAN__HARVEST__MQ__HOSTNAME=redis
|
|
||||||
CKAN__HARVEST__MQ__PORT=6379
|
|
||||||
CKAN__HARVEST__MQ__REDIS_DB=1
|
|
||||||
|
|
||||||
|
|
||||||
docker-compose.yml
|
|
||||||
DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
PGDATA=/var/lib/postgresql/data/db
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
--- ckan/ckan/model/__init__.py 2021-02-16 14:47:06.168327441 +0100
|
|
||||||
+++ ckan/ckan/model/__init__.py 2021-02-16 14:48:00.740780218 +0100
|
|
||||||
@@ -266,7 +266,7 @@
|
|
||||||
self.reset_alembic_output()
|
|
||||||
alembic_config = AlembicConfig(self._alembic_ini)
|
|
||||||
alembic_config.set_main_option(
|
|
||||||
- "sqlalchemy.url", str(self.metadata.bind.url)
|
|
||||||
+ "sqlalchemy.url", str(self.metadata.bind.url).replace('%', '%%')
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
sqlalchemy_migrate_version = self.metadata.bind.execute(
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
--- ckan/ckan/logic/action/update.py 2021-02-17 16:46:55.673578728 +0100
|
|
||||||
+++ ckan/ckan/logic/action/update-edit.py 2021-02-17 16:47:28.905879170 +0100
|
|
||||||
@@ -929,7 +929,7 @@
|
|
||||||
|
|
||||||
'''
|
|
||||||
model = context['model']
|
|
||||||
- session = model.Session
|
|
||||||
+ session = model.meta.create_local_session()
|
|
||||||
context['session'] = session
|
|
||||||
|
|
||||||
user = context['user']
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
--- ckan/ckanext/datastore/backend/postgres.py 2021-02-18 11:01:56.692267462 +0100
|
|
||||||
+++ ckan/ckanext/datastore/backend/postgres-patch.py 2021-02-18 13:45:16.033193435 +0100
|
|
||||||
@@ -1690,7 +1690,7 @@
|
|
||||||
read only user.
|
|
||||||
'''
|
|
||||||
write_connection = self._get_write_engine().connect()
|
|
||||||
- read_connection_user = sa_url.make_url(self.read_url).username
|
|
||||||
+ read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0]
|
|
||||||
|
|
||||||
drop_foo_sql = u'DROP TABLE IF EXISTS _foo'
|
|
||||||
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
shopt -s nullglob
|
|
||||||
for patch in patches/*.patch; do
|
|
||||||
/usr/bin/patch -p0 -i $patch
|
|
||||||
done
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
http-socket = :5000
|
|
||||||
uid = ckan
|
|
||||||
guid = ckan
|
|
||||||
plugins = python3
|
|
||||||
wsgi-file = /srv/app/wsgi.py
|
|
||||||
virtualenv = /srv/app
|
|
||||||
module = wsgi:application
|
|
||||||
master = true
|
|
||||||
processes = 5
|
|
||||||
pidfile = /tmp/%n.pid
|
|
||||||
harakiri = 50
|
|
||||||
max-requests = 5000
|
|
||||||
vacuum = true
|
|
||||||
callable = application
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
http = 0.0.0.0:8800
|
|
||||||
uid = www-data
|
|
||||||
guid = www-data
|
|
||||||
wsgi-file = /srv/app/datapusher.wsgi
|
|
||||||
master = true
|
|
||||||
socket = /tmp/uwsgi.sock
|
|
||||||
plugins = http,python
|
|
||||||
pidfile = /tmp/%n.pid
|
|
||||||
harakiri = 50
|
|
||||||
max-requests = 5000
|
|
||||||
vacuum = true
|
|
||||||
callable = application
|
|
||||||
buffer-size = 32768
|
|
||||||
|
|
@ -1,37 +0,0 @@
|
||||||
import os
|
|
||||||
import uuid
|
|
||||||
|
|
||||||
DEBUG = False
|
|
||||||
TESTING = False
|
|
||||||
SECRET_KEY = str(uuid.uuid4())
|
|
||||||
USERNAME = str(uuid.uuid4())
|
|
||||||
PASSWORD = str(uuid.uuid4())
|
|
||||||
|
|
||||||
NAME = 'datapusher'
|
|
||||||
|
|
||||||
# Webserver host and port
|
|
||||||
|
|
||||||
HOST = os.environ.get('DATAPUSHER_HOST', '0.0.0.0')
|
|
||||||
PORT = os.environ.get('DATAPUSHER_PORT', 8800)
|
|
||||||
|
|
||||||
# Database
|
|
||||||
|
|
||||||
SQLALCHEMY_DATABASE_URI = os.environ.get('DATAPUSHER_SQLALCHEMY_DATABASE_URI', 'sqlite:////tmp/job_store.db')
|
|
||||||
|
|
||||||
# Download and streaming settings
|
|
||||||
|
|
||||||
MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '1024000'))
|
|
||||||
CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384'))
|
|
||||||
CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250'))
|
|
||||||
DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30'))
|
|
||||||
|
|
||||||
# Verify SSL
|
|
||||||
SSL_VERIFY = os.environ.get('DATAPUSHER_SSL_VERIFY', True)
|
|
||||||
|
|
||||||
# logging
|
|
||||||
#LOG_FILE = '/tmp/ckan_service.log'
|
|
||||||
STDERR = True
|
|
||||||
|
|
||||||
# Rewrite resource URL's when ckan callback url base is used
|
|
||||||
REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', True)
|
|
||||||
REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/')
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
|
|
||||||
user nginx;
|
|
||||||
worker_processes auto;
|
|
||||||
|
|
||||||
error_log /var/log/nginx/error.log warn;
|
|
||||||
pid /var/run/nginx.pid;
|
|
||||||
|
|
||||||
|
|
||||||
events {
|
|
||||||
worker_connections 1024;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
http {
|
|
||||||
include /etc/nginx/mime.types;
|
|
||||||
default_type application/octet-stream;
|
|
||||||
|
|
||||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
|
||||||
'$status $body_bytes_sent "$http_referer" '
|
|
||||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
|
||||||
|
|
||||||
access_log /var/log/nginx/access.log main;
|
|
||||||
|
|
||||||
sendfile on;
|
|
||||||
#tcp_nopush on;
|
|
||||||
|
|
||||||
keepalive_timeout 65;
|
|
||||||
|
|
||||||
#gzip on;
|
|
||||||
|
|
||||||
include /etc/nginx/sites-enabled/*.conf;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
194
setup/prerun.py
194
setup/prerun.py
|
|
@ -1,194 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import subprocess
|
|
||||||
import psycopg2
|
|
||||||
import urllib3
|
|
||||||
import time
|
|
||||||
import re
|
|
||||||
|
|
||||||
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
|
|
||||||
|
|
||||||
RETRY = 5
|
|
||||||
|
|
||||||
def update_plugins():
|
|
||||||
|
|
||||||
plugins = os.environ.get('CKAN__PLUGINS', '')
|
|
||||||
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
|
|
||||||
print(plugins)
|
|
||||||
cmd = ['ckan', 'config-tool', ckan_ini,
|
|
||||||
'ckan.plugins = {}'.format(plugins)]
|
|
||||||
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
|
||||||
print('[prerun] Plugins set.')
|
|
||||||
|
|
||||||
|
|
||||||
def check_main_db_connection(retry=None):
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
|
|
||||||
return check_db_connection(conn_str, retry)
|
|
||||||
|
|
||||||
|
|
||||||
def check_datastore_db_connection(retry=None):
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
|
|
||||||
return check_db_connection(conn_str, retry)
|
|
||||||
|
|
||||||
|
|
||||||
def check_db_connection(conn_str, retry=None):
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print(str(e))
|
|
||||||
print('[prerun] Unable to connect to the database, waiting...')
|
|
||||||
time.sleep(10)
|
|
||||||
check_db_connection(conn_str, retry=retry - 1)
|
|
||||||
else:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def check_solr_connection(retry=None):
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
url = os.environ.get('CKAN_SOLR_URL', '')
|
|
||||||
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
|
||||||
http = urllib3.PoolManager()
|
|
||||||
try:
|
|
||||||
r = http.request('GET', search_url)
|
|
||||||
except urllib3.exceptions.ConnectionError as e:
|
|
||||||
print(str(e))
|
|
||||||
print('[prerun] Unable to connect to solr, waiting...')
|
|
||||||
time.sleep(10)
|
|
||||||
check_solr_connection(retry=retry - 1)
|
|
||||||
else:
|
|
||||||
print('[prerun] Connection Status from SOLR is ', (r.status))
|
|
||||||
|
|
||||||
def init_db():
|
|
||||||
|
|
||||||
db_command = ['ckan', '-c', ckan_ini,
|
|
||||||
'db', 'init']
|
|
||||||
print('[prerun] Initializing or upgrading db - start')
|
|
||||||
try:
|
|
||||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
|
||||||
print('[prerun] Initializing or upgrading db - end')
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print(e.output)
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print(e.output)
|
|
||||||
raise e
|
|
||||||
|
|
||||||
|
|
||||||
def init_datastore_db():
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] Skipping datastore initialization')
|
|
||||||
return
|
|
||||||
|
|
||||||
datastore_perms_command = ['ckan', '-c', ckan_ini,
|
|
||||||
'datastore', 'set-permissions']
|
|
||||||
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
cursor = connection.cursor()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - start')
|
|
||||||
try:
|
|
||||||
datastore_perms = subprocess.Popen(
|
|
||||||
datastore_perms_command,
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
perms_sql = datastore_perms.stdout.read().decode('utf-8')
|
|
||||||
# Remove internal pg command as psycopg2 does not like it
|
|
||||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
|
||||||
cursor.execute(perms_sql)
|
|
||||||
for notice in connection.notices:
|
|
||||||
print(notice)
|
|
||||||
|
|
||||||
connection.commit()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - end')
|
|
||||||
print(datastore_perms.stdout.read().decode('utf-8'))
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print('[prerun] Could not initialize datastore')
|
|
||||||
print(str(e))
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print(e.output)
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print(e.output)
|
|
||||||
raise e
|
|
||||||
finally:
|
|
||||||
cursor.close()
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def create_sysadmin():
|
|
||||||
|
|
||||||
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
|
||||||
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
|
||||||
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
|
||||||
|
|
||||||
if name and password and email:
|
|
||||||
|
|
||||||
# Check if user exists
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
|
|
||||||
|
|
||||||
out = subprocess.check_output(command)
|
|
||||||
if 'User:None' not in re.sub(r'\s', '', out.decode()):
|
|
||||||
print('[prerun] Sysadmin user exists, skipping creation')
|
|
||||||
return
|
|
||||||
|
|
||||||
# Create user
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'user', 'add',
|
|
||||||
name,
|
|
||||||
'password=' + password,
|
|
||||||
'email=' + email]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print('[prerun] Created user {0}'.format(name))
|
|
||||||
|
|
||||||
# Make it sysadmin
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
|
|
||||||
name]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print('[prerun] Made user {0} a sysadmin'.format(name))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
|
||||||
|
|
||||||
if maintenance:
|
|
||||||
print('[prerun] Maintenance mode, skipping setup...')
|
|
||||||
else:
|
|
||||||
check_main_db_connection()
|
|
||||||
init_db()
|
|
||||||
update_plugins()
|
|
||||||
check_datastore_db_connection()
|
|
||||||
init_datastore_db()
|
|
||||||
check_solr_connection()
|
|
||||||
create_sysadmin()
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m;
|
|
||||||
proxy_temp_path /tmp/nginx_proxy 1 2;
|
|
||||||
|
|
||||||
server {
|
|
||||||
client_max_body_size 100M;
|
|
||||||
location / {
|
|
||||||
proxy_pass http://ckan:5000/;
|
|
||||||
proxy_set_header X-Forwarded-For $remote_addr;
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_cache cache;
|
|
||||||
proxy_cache_bypass $cookie_auth_tkt;
|
|
||||||
proxy_no_cache $cookie_auth_tkt;
|
|
||||||
proxy_cache_valid 30m;
|
|
||||||
proxy_cache_key $host$scheme$proxy_host$request_uri;
|
|
||||||
# In emergency comment out line to force caching
|
|
||||||
# proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,46 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Run the prerun script to init CKAN and create the default admin user
|
|
||||||
sudo -u ckan -EH python3 prerun.py
|
|
||||||
|
|
||||||
# Run any startup scripts provided by images extending this one
|
|
||||||
if [[ -d "/docker-entrypoint.d" ]]
|
|
||||||
then
|
|
||||||
for f in /docker-entrypoint.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
|
||||||
if [ $? -eq 0 ]
|
|
||||||
then
|
|
||||||
if [ "$PASSWORD_PROTECT" = true ]
|
|
||||||
then
|
|
||||||
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
|
||||||
then
|
|
||||||
# Generate htpasswd file for basicauth
|
|
||||||
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
# Start uwsgi with basicauth
|
|
||||||
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
|
|
||||||
else
|
|
||||||
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf &
|
|
||||||
# Start uwsgi
|
|
||||||
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "[prerun] failed...not starting CKAN."
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
echo "[start_datapusher.sh] Starting supervisord."
|
|
||||||
# Start supervisord
|
|
||||||
supervisord --configuration /etc/supervisord.conf
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
[program:datapusher-uwsgi]
|
|
||||||
command=/usr/sbin/uwsgi -i /srv/app/datapusher-uwsgi.ini
|
|
||||||
priority=501
|
|
||||||
autostart=true
|
|
||||||
autorestart=true
|
|
||||||
redirect_stderr=true
|
|
||||||
stdout_logfile=/dev/stdout
|
|
||||||
stdout_logfile_maxbytes=0
|
|
||||||
stderr_logfile=/dev/stdout
|
|
||||||
stderr_logfile_maxbytes=0
|
|
||||||
user=www-data
|
|
||||||
environment=HOME="/srv/app",USER="www-data"
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
[program:ckan-worker]
|
|
||||||
command=ckan -c /srv/app/ckan.ini jobs worker
|
|
||||||
priority=501
|
|
||||||
autostart=true
|
|
||||||
autorestart=true
|
|
||||||
redirect_stderr=true
|
|
||||||
stdout_logfile=/dev/stdout
|
|
||||||
stdout_logfile_maxbytes=0
|
|
||||||
stderr_logfile=/dev/stdout
|
|
||||||
stderr_logfile_maxbytes=0
|
|
||||||
user=ckan
|
|
||||||
environment=HOME="/srv/app",USER="ckan"
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
[unix_http_server]
|
|
||||||
file = /tmp/supervisor.sock
|
|
||||||
chmod = 0777
|
|
||||||
chown = nobody:nogroup
|
|
||||||
|
|
||||||
[supervisord]
|
|
||||||
logfile = /tmp/supervisord.log
|
|
||||||
logfile_maxbytes = 50MB
|
|
||||||
logfile_backups=10
|
|
||||||
loglevel = trace
|
|
||||||
pidfile = /tmp/supervisord.pid
|
|
||||||
nodaemon = true
|
|
||||||
umask = 022
|
|
||||||
identifier = supervisor
|
|
||||||
|
|
||||||
[supervisorctl]
|
|
||||||
serverurl = unix:///tmp/supervisor.sock
|
|
||||||
|
|
||||||
[rpcinterface:supervisor]
|
|
||||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
|
||||||
|
|
||||||
[include]
|
|
||||||
files = /etc/supervisord.d/*.conf
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
import os
|
|
||||||
from ckan.config.middleware import make_app
|
|
||||||
from ckan.cli import CKANConfigLoader
|
|
||||||
from logging.config import fileConfig as loggingFileConfig
|
|
||||||
config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
|
|
||||||
abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
|
|
||||||
loggingFileConfig(config_filepath)
|
|
||||||
config = CKANConfigLoader(config_filepath).get_config()
|
|
||||||
application = make_app(config)
|
|
||||||
|
|
@ -2,10 +2,9 @@ FROM solr:6.6.6
|
||||||
|
|
||||||
# Enviroment variables
|
# Enviroment variables
|
||||||
ENV SOLR_CORE ckan
|
ENV SOLR_CORE ckan
|
||||||
|
ENV SOLR_VERSION 6.6.6
|
||||||
# Build Arguments
|
ENV CKAN_VERSION 2.9.1
|
||||||
ARG CKAN_VERSION
|
###TODO!!! CKAN_VERSION to be passed in as an ARG
|
||||||
ARG SOLR_VERSION
|
|
||||||
|
|
||||||
# root user for initial config
|
# root user for initial config
|
||||||
USER root
|
USER root
|
||||||
|
|
@ -16,14 +15,14 @@ RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \
|
||||||
mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index
|
mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index
|
||||||
|
|
||||||
# Add files
|
# Add files
|
||||||
COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml
|
ADD solrconfig.xml \
|
||||||
ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
|
https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \
|
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
|
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
|
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
|
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
|
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
|
||||||
/opt/solr/server/solr/${SOLR_CORE}/conf/
|
/opt/solr/server/solr/$SOLR_CORE/conf/
|
||||||
|
|
||||||
# Create core.properties
|
# Create core.properties
|
||||||
RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties
|
RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties
|
||||||
|
|
@ -33,4 +32,3 @@ RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE}
|
||||||
|
|
||||||
# non-root user for runtime
|
# non-root user for runtime
|
||||||
USER ${SOLR_USER}:${SOLR_USER}
|
USER ${SOLR_USER}:${SOLR_USER}
|
||||||
|
|
||||||
|
|
|
||||||
12
todo.txt
12
todo.txt
|
|
@ -1,12 +0,0 @@
|
||||||
Get the NGINX container running <done>
|
|
||||||
Add the redis container <done>
|
|
||||||
Add the solr container <done>
|
|
||||||
Add the db container <done>
|
|
||||||
Get a datapusher container working with supervisord <done>
|
|
||||||
Test the datapusher container works
|
|
||||||
Take out ARG's from docker-compose.yml is not needed
|
|
||||||
Test with new versions of CKAN (ie: if problems with Python 3.9)
|
|
||||||
Create (and push) the datapusher image
|
|
||||||
Create the ckan container
|
|
||||||
Create the ckanext container
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue