Add ckan patch and minor version
This commit is contained in:
Kiril-Poposki1998
parent
f696c00a04
commit
1dc7282a51
|
|
@ -9,7 +9,9 @@ volumes:
|
||||||
services:
|
services:
|
||||||
ckan:
|
ckan:
|
||||||
container_name: ckan
|
container_name: ckan
|
||||||
image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION}
|
build:
|
||||||
|
context: ../../images/ckan/2.10
|
||||||
|
dockerfile: Dockerfile
|
||||||
networks:
|
networks:
|
||||||
- frontend
|
- frontend
|
||||||
- backend
|
- backend
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
# Runtime configuration of CKAN enabled through ckanext-envvars
|
||||||
|
# Information about how it works: https://github.com/okfn/ckanext-envvars
|
||||||
|
# Note that variables here take presedence over build/up time variables in .env
|
||||||
|
|
||||||
|
# Set to true to disable CKAN from starting and serve a maintenance page
|
||||||
|
MAINTENANCE_MODE=false
|
||||||
|
|
||||||
|
# General Settings
|
||||||
|
CKAN_SITE_ID=default
|
||||||
|
CKAN_SITE_URL=http://localhost:5000
|
||||||
|
CKAN_PORT=5000
|
||||||
|
CKAN_MAX_UPLOAD_SIZE_MB=20
|
||||||
|
CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
|
||||||
|
# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
|
||||||
|
CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
|
||||||
|
CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
|
||||||
|
# CKAN Plugins
|
||||||
|
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
|
||||||
|
# CKAN requires storage path to be set in order for filestore to be enabled
|
||||||
|
CKAN__STORAGE_PATH=/srv/app/data
|
||||||
|
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
|
||||||
|
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
|
||||||
|
CKAN_SYSADMIN_NAME=sysadmin
|
||||||
|
CKAN_SYSADMIN_PASSWORD=password
|
||||||
|
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
|
||||||
|
|
||||||
|
# Email settings
|
||||||
|
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
|
||||||
|
CKAN_SMTP_STARTTLS=True
|
||||||
|
CKAN_SMTP_USER=user
|
||||||
|
CKAN_SMTP_PASSWORD=pass
|
||||||
|
CKAN_SMTP_MAIL_FROM=ckan@localhost
|
||||||
|
|
||||||
|
# Datapusher configuration
|
||||||
|
CKAN__DATAPUSHER__URL=http://datapusher:8000
|
||||||
|
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
|
||||||
|
CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0
|
||||||
|
# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts
|
||||||
|
|
||||||
|
# Solr configuration
|
||||||
|
CKAN_VERSION=2.10
|
||||||
|
CKAN_CORE_NAME=ckan
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
# Variables in this file will be used as build arguments when running
|
||||||
|
# docker-compose build and docker-compose up
|
||||||
|
# Verify correct substitution with "docker-compose config"
|
||||||
|
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
|
||||||
|
# docker-compose down -v
|
||||||
|
# docker-compose build
|
||||||
|
# docker-compose up -d
|
||||||
|
|
||||||
|
# Database
|
||||||
|
POSTGRES_PASSWORD=ckan
|
||||||
|
POSTGRES_PORT=5432
|
||||||
|
DATASTORE_READONLY_PASSWORD=datastore
|
||||||
|
|
||||||
|
# CKAN
|
||||||
|
CKAN_VERSION=2.10.4
|
||||||
|
CKAN_SITE_ID=default
|
||||||
|
CKAN_SITE_URL=http://localhost:5000
|
||||||
|
CKAN_PORT=5000
|
||||||
|
CKAN_MAX_UPLOAD_SIZE_MB=10
|
||||||
|
|
||||||
|
# Datapusher
|
||||||
|
DATAPUSHER_VERSION=0.0.17
|
||||||
|
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
|
||||||
|
DATAPUSHER_CHUNK_SIZE=16384
|
||||||
|
DATAPUSHER_CHUNK_INSERT_ROWS=250
|
||||||
|
DATAPUSHER_DOWNLOAD_TIMEOUT=30
|
||||||
|
DATAPUSHER_SSL_VERIFY=False
|
||||||
|
DATAPUSHER_REWRITE_RESOURCES=True
|
||||||
|
DATAPUSHER_REWRITE_URL=http://ckan:5005
|
||||||
|
|
||||||
|
# SOLR
|
||||||
|
CKAN_CORE_NAME=ckan
|
||||||
|
|
||||||
|
# Redis
|
||||||
|
REDIS_VERSION=6.0.7
|
||||||
|
|
@ -0,0 +1,98 @@
|
||||||
|
# docker-compose build && docker-compose up -d
|
||||||
|
version: "3.8"
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
ckan_data:
|
||||||
|
pg_data:
|
||||||
|
solr_data:
|
||||||
|
|
||||||
|
services:
|
||||||
|
ckan:
|
||||||
|
container_name: ckan
|
||||||
|
build:
|
||||||
|
context: ../../images/ckan/2.11/
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
networks:
|
||||||
|
- frontend
|
||||||
|
- backend
|
||||||
|
depends_on:
|
||||||
|
- db
|
||||||
|
- solr
|
||||||
|
ports:
|
||||||
|
- "0.0.0.0:${CKAN_PORT}:5000"
|
||||||
|
env_file:
|
||||||
|
- ./.ckan-env
|
||||||
|
environment:
|
||||||
|
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
||||||
|
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
||||||
|
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
||||||
|
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
||||||
|
- CKAN_REDIS_URL=redis://redis:6379/1
|
||||||
|
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
||||||
|
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
||||||
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
||||||
|
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- ckan_data:/srv/app/data
|
||||||
|
|
||||||
|
datapusher:
|
||||||
|
container_name: datapusher
|
||||||
|
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
|
||||||
|
networks:
|
||||||
|
- frontend
|
||||||
|
- backend
|
||||||
|
ports:
|
||||||
|
- "8000:8000"
|
||||||
|
environment:
|
||||||
|
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
|
||||||
|
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
|
||||||
|
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
|
||||||
|
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
|
||||||
|
- DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
|
||||||
|
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
|
||||||
|
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
|
||||||
|
|
||||||
|
|
||||||
|
db:
|
||||||
|
container_name: db
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: postgresql/Dockerfile
|
||||||
|
args:
|
||||||
|
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
||||||
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
||||||
|
networks:
|
||||||
|
- backend
|
||||||
|
environment:
|
||||||
|
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
||||||
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
||||||
|
- PGDATA=/var/lib/postgresql/data/db
|
||||||
|
volumes:
|
||||||
|
- pg_data:/var/lib/postgresql/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "pg_isready", "-U", "ckan"]
|
||||||
|
|
||||||
|
solr:
|
||||||
|
container_name: solr
|
||||||
|
image: solr:8.11.1
|
||||||
|
networks:
|
||||||
|
- backend
|
||||||
|
env_file:
|
||||||
|
- ./.ckan-env
|
||||||
|
environment:
|
||||||
|
- CKAN_CORE_NAME=${CKAN_CORE_NAME}
|
||||||
|
- CKAN_VERSION=${CKAN_VERSION}
|
||||||
|
volumes:
|
||||||
|
- solr_data:/var/solr
|
||||||
|
- ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
|
||||||
|
|
||||||
|
redis:
|
||||||
|
container_name: redis
|
||||||
|
image: redis:${REDIS_VERSION}
|
||||||
|
networks:
|
||||||
|
- backend
|
||||||
|
|
||||||
|
networks:
|
||||||
|
frontend:
|
||||||
|
backend:
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
FROM postgis/postgis:14-3.2-alpine
|
||||||
|
|
||||||
|
# Allow connections; we don't map out any ports so only linked docker containers can connect
|
||||||
|
RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf
|
||||||
|
|
||||||
|
# Customize default user/pass/db
|
||||||
|
ENV POSTGRES_DB ckan
|
||||||
|
ENV POSTGRES_USER ckan
|
||||||
|
ARG POSTGRES_PASSWORD
|
||||||
|
ARG DS_RO_PASS
|
||||||
|
|
||||||
|
# Include datastore setup scripts
|
||||||
|
COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
|
||||||
|
CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
|
||||||
|
CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
|
||||||
|
GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;
|
||||||
|
EOSQL
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
ALTER VIEW geometry_columns OWNER TO ckan;
|
||||||
|
ALTER TABLE spatial_ref_sys OWNER TO ckan;
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Initialize SOLR for CKAN by creating a ckan core
|
||||||
|
# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION
|
||||||
|
# Example:
|
||||||
|
# CKAN_CORE_NAME=ckan
|
||||||
|
# CKAN_VERSION=2.9.7
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
MAJOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 1`
|
||||||
|
MINOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 2`
|
||||||
|
|
||||||
|
CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/`echo $MAJOR_VERSION.$MINOR_VERSION`/ckan/config/solr/schema.xml
|
||||||
|
|
||||||
|
echo "Check whether managed schema exists for CKAN $CKAN_VERSION"
|
||||||
|
if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then
|
||||||
|
echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Check whether SOLR is initialized for CKAN"
|
||||||
|
CORESDIR=/var/solr/data
|
||||||
|
|
||||||
|
COREDIR="$CORESDIR/$CKAN_CORE_NAME"
|
||||||
|
if [ -d "$COREDIR" ]; then
|
||||||
|
echo "SOLR already initialized, skipping initialization"
|
||||||
|
else
|
||||||
|
echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION"
|
||||||
|
# init script for handling an empty /var/solr
|
||||||
|
/opt/docker-solr/scripts/init-var-solr
|
||||||
|
|
||||||
|
# Precreate CKAN core
|
||||||
|
/opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME
|
||||||
|
|
||||||
|
# Replace the managed schema with CKANs schema
|
||||||
|
echo "Adding CKAN managed schema"
|
||||||
|
curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s
|
||||||
|
|
||||||
|
echo "SOLR initialized"
|
||||||
|
fi
|
||||||
|
|
@ -4,11 +4,11 @@
|
||||||
FROM alpine:3.17.2 as ckanbuild
|
FROM alpine:3.17.2 as ckanbuild
|
||||||
|
|
||||||
# Used by Github Actions to tag the image with
|
# Used by Github Actions to tag the image with
|
||||||
ENV IMAGE_TAG=2.10.4
|
ENV IMAGE_TAG=2.10.5
|
||||||
|
|
||||||
# Set CKAN version to build
|
# Set CKAN version to build
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
ENV GIT_URL=https://github.com/ckan/ckan.git
|
||||||
ENV GIT_BRANCH=ckan-2.10.4
|
ENV GIT_BRANCH=ckan-2.10.5
|
||||||
|
|
||||||
# Set src dirs
|
# Set src dirs
|
||||||
ENV SRC_DIR=/srv/app/src
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
|
|
||||||
|
|
@ -4,11 +4,11 @@
|
||||||
FROM ubuntu:focal-20210827 as ckanbuild
|
FROM ubuntu:focal-20210827 as ckanbuild
|
||||||
|
|
||||||
# Used by Github Actions to tag the image with
|
# Used by Github Actions to tag the image with
|
||||||
ENV IMAGE_TAG=2.10.4-focal
|
ENV IMAGE_TAG=2.10.5-focal
|
||||||
|
|
||||||
# Set CKAN version to build
|
# Set CKAN version to build
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
ENV GIT_URL=https://github.com/ckan/ckan.git
|
||||||
ENV GIT_BRANCH=ckan-2.10.4
|
ENV GIT_BRANCH=ckan-2.10.5
|
||||||
|
|
||||||
# Set timezone
|
# Set timezone
|
||||||
ENV TZ=UTC
|
ENV TZ=UTC
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,190 @@
|
||||||
|
##################
|
||||||
|
### Build CKAN ###
|
||||||
|
##################
|
||||||
|
FROM alpine:3.17.2 as ckanbuild
|
||||||
|
|
||||||
|
# Used by Github Actions to tag the image with
|
||||||
|
ENV IMAGE_TAG=2.11
|
||||||
|
|
||||||
|
# Set CKAN version to build
|
||||||
|
ENV GIT_URL=https://github.com/ckan/ckan.git
|
||||||
|
ENV GIT_BRANCH=ckan-2.11
|
||||||
|
|
||||||
|
# Set src dirs
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
|
||||||
|
WORKDIR ${SRC_DIR}
|
||||||
|
|
||||||
|
# Packages to build CKAN requirements and plugins
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
python3 \
|
||||||
|
python3-dev \
|
||||||
|
git \
|
||||||
|
curl \
|
||||||
|
postgresql-dev \
|
||||||
|
linux-headers \
|
||||||
|
gcc \
|
||||||
|
make \
|
||||||
|
g++ \
|
||||||
|
autoconf \
|
||||||
|
automake \
|
||||||
|
libtool \
|
||||||
|
patch \
|
||||||
|
musl-dev \
|
||||||
|
pcre-dev \
|
||||||
|
pcre \
|
||||||
|
libffi-dev \
|
||||||
|
libxml2-dev \
|
||||||
|
libxslt-dev
|
||||||
|
|
||||||
|
# Create the src directory
|
||||||
|
RUN mkdir -p ${SRC_DIR}
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
||||||
|
python ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
# Downgrade setuptools so that CKAN requirements can be built
|
||||||
|
RUN pip install setuptools==44.1.0
|
||||||
|
|
||||||
|
# Fetch and build CKAN and requirements
|
||||||
|
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
||||||
|
# Copy patches and apply patches script
|
||||||
|
COPY ./patches ${SRC_DIR}/patches
|
||||||
|
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
|
||||||
|
# Apply patches
|
||||||
|
# RUN ${SRC_DIR}/apply_ckan_patches.sh
|
||||||
|
RUN rm -rf /srv/app/src/ckan/.git
|
||||||
|
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
|
||||||
|
RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2
|
||||||
|
|
||||||
|
|
||||||
|
###########################
|
||||||
|
### Default-Extensions ####
|
||||||
|
###########################
|
||||||
|
FROM alpine:3.17.2 as extbuild
|
||||||
|
|
||||||
|
# Set src dirs
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
|
||||||
|
# List of default extensions
|
||||||
|
ENV DEFAULT_EXTENSIONS envvars
|
||||||
|
|
||||||
|
# Locations and tags, please use specific tags or revisions
|
||||||
|
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
|
||||||
|
ENV ENVVARS_GIT_BRANCH=0.0.2
|
||||||
|
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
python3 \
|
||||||
|
python3-dev \
|
||||||
|
git \
|
||||||
|
curl
|
||||||
|
|
||||||
|
# Create the src directory
|
||||||
|
RUN mkdir -p ${SRC_DIR}
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
||||||
|
python ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
# Fetch and build the default CKAN extensions
|
||||||
|
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
|
||||||
|
|
||||||
|
############
|
||||||
|
### MAIN ###
|
||||||
|
############
|
||||||
|
FROM alpine:3.17.2
|
||||||
|
|
||||||
|
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
||||||
|
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
|
||||||
|
|
||||||
|
ENV APP_DIR=/srv/app
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV CKAN_DIR=${SRC_DIR}/ckan
|
||||||
|
ENV DATA_DIR=/srv/app/data
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
ENV CKAN_SITE_URL=http://localhost:5000
|
||||||
|
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
|
||||||
|
|
||||||
|
# Install necessary packages to run CKAN
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
python3 \
|
||||||
|
bash \
|
||||||
|
git \
|
||||||
|
gettext \
|
||||||
|
curl \
|
||||||
|
postgresql-client \
|
||||||
|
libmagic \
|
||||||
|
pcre \
|
||||||
|
libxslt \
|
||||||
|
libxml2 \
|
||||||
|
tzdata \
|
||||||
|
apache2-utils && \
|
||||||
|
# Create SRC_DIR
|
||||||
|
mkdir -p ${SRC_DIR}
|
||||||
|
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
||||||
|
python ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
# Get artifacts from build stages
|
||||||
|
COPY --from=ckanbuild /wheels /srv/app/wheels
|
||||||
|
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
||||||
|
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
|
||||||
|
|
||||||
|
# Additional install steps for build stages artifacts
|
||||||
|
RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2
|
||||||
|
|
||||||
|
# Create a local user and group to run the app
|
||||||
|
RUN addgroup -g 92 -S ckan && \
|
||||||
|
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
||||||
|
|
||||||
|
WORKDIR ${CKAN_DIR}
|
||||||
|
|
||||||
|
# Install CKAN
|
||||||
|
RUN pip install -e /srv/app/src/ckan && \
|
||||||
|
cp who.ini ${APP_DIR} && \
|
||||||
|
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
|
||||||
|
# Install default CKAN extensions
|
||||||
|
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
|
||||||
|
# Create and update CKAN config
|
||||||
|
# Set timezone
|
||||||
|
echo "UTC" > /etc/timezone && \
|
||||||
|
# Generate CKAN config
|
||||||
|
ckan generate config ${APP_DIR}/production.ini && \
|
||||||
|
ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
|
||||||
|
# Configure plugins
|
||||||
|
ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
||||||
|
# Create the data directory
|
||||||
|
mkdir ${DATA_DIR} && \
|
||||||
|
# Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created
|
||||||
|
ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \
|
||||||
|
# Set the default level for extensions to INFO
|
||||||
|
ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
|
||||||
|
# Change ownership to app user
|
||||||
|
chown -R ckan:ckan /srv/app
|
||||||
|
|
||||||
|
# Remove wheels
|
||||||
|
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
|
||||||
|
|
||||||
|
# Copy necessary scripts
|
||||||
|
COPY setup/app ${APP_DIR}
|
||||||
|
|
||||||
|
WORKDIR ${APP_DIR}
|
||||||
|
|
||||||
|
# Create entrypoint directory for children image scripts
|
||||||
|
ONBUILD RUN mkdir docker-entrypoint.d
|
||||||
|
|
||||||
|
# Create afterinit directory for children image scripts
|
||||||
|
ONBUILD RUN mkdir docker-afterinit.d
|
||||||
|
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
||||||
|
|
||||||
|
USER ckan
|
||||||
|
|
||||||
|
CMD ["/srv/app/start_ckan.sh"]
|
||||||
|
|
@ -0,0 +1,242 @@
|
||||||
|
##################
|
||||||
|
### Build CKAN ###
|
||||||
|
##################
|
||||||
|
FROM ubuntu:focal-20210827 as ckanbuild
|
||||||
|
|
||||||
|
# Used by Github Actions to tag the image with
|
||||||
|
ENV IMAGE_TAG=2.10-focal
|
||||||
|
|
||||||
|
# Set CKAN version to build
|
||||||
|
ENV GIT_URL=https://github.com/ckan/ckan.git
|
||||||
|
ENV GIT_BRANCH=ckan-2.11
|
||||||
|
|
||||||
|
# Set timezone
|
||||||
|
ENV TZ=UTC
|
||||||
|
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||||
|
|
||||||
|
# Set Locale
|
||||||
|
ENV LC_ALL=en_US.UTF-8
|
||||||
|
|
||||||
|
# Set src dirs
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
|
||||||
|
WORKDIR ${SRC_DIR}
|
||||||
|
|
||||||
|
# Set the locale
|
||||||
|
RUN apt-get update
|
||||||
|
RUN apt-get install --no-install-recommends -y locales
|
||||||
|
RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
|
||||||
|
RUN dpkg-reconfigure --frontend=noninteractive locales
|
||||||
|
RUN update-locale LANG=${LC_ALL}
|
||||||
|
|
||||||
|
# Instal apt-utils
|
||||||
|
RUN apt-get install --no-install-recommends -y \
|
||||||
|
apt-utils
|
||||||
|
|
||||||
|
# Packages to build CKAN requirements and plugins
|
||||||
|
RUN apt-get install --no-install-recommends -y \
|
||||||
|
git \
|
||||||
|
curl \
|
||||||
|
ca-certificates \
|
||||||
|
python3 \
|
||||||
|
libpq-dev \
|
||||||
|
linux-headers-generic \
|
||||||
|
gcc-10 \
|
||||||
|
make \
|
||||||
|
g++-10 \
|
||||||
|
autoconf \
|
||||||
|
automake \
|
||||||
|
libtool \
|
||||||
|
patch \
|
||||||
|
libpcre3-dev \
|
||||||
|
libpcre3 \
|
||||||
|
python3-dev \
|
||||||
|
libffi-dev \
|
||||||
|
libxml2-dev \
|
||||||
|
libxslt-dev
|
||||||
|
|
||||||
|
# Use gcc 10
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10
|
||||||
|
|
||||||
|
# Link python to python3
|
||||||
|
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||||
|
|
||||||
|
# Create the src directory
|
||||||
|
RUN mkdir -p ${SRC_DIR}
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
||||||
|
python ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
# Downgrade setuptools so that CKAN requirements can be built
|
||||||
|
RUN pip install setuptools==44.1.0
|
||||||
|
|
||||||
|
# Fetch and build CKAN and requirements
|
||||||
|
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
||||||
|
# Copy patches and apply patches script
|
||||||
|
COPY ./patches ${SRC_DIR}/patches
|
||||||
|
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
|
||||||
|
# Apply patches
|
||||||
|
RUN ${SRC_DIR}/apply_ckan_patches.sh
|
||||||
|
RUN rm -rf /srv/app/src/ckan/.git
|
||||||
|
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
|
||||||
|
RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2
|
||||||
|
|
||||||
|
###########################
|
||||||
|
### Default-Extensions ####
|
||||||
|
###########################
|
||||||
|
FROM ubuntu:focal-20210827 as extbuild
|
||||||
|
|
||||||
|
# Set src dirs
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
|
||||||
|
# List of default extensions
|
||||||
|
ENV DEFAULT_EXTENSIONS envvars
|
||||||
|
|
||||||
|
# Locations and tags, please use specific tags or revisions
|
||||||
|
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
|
||||||
|
ENV ENVVARS_GIT_BRANCH=0.0.1
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install --no-install-recommends -y \
|
||||||
|
git \
|
||||||
|
curl \
|
||||||
|
ca-certificates \
|
||||||
|
python3 \
|
||||||
|
python3-dev
|
||||||
|
|
||||||
|
# Link python to python3
|
||||||
|
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||||
|
|
||||||
|
# Create the src directory
|
||||||
|
RUN mkdir -p ${SRC_DIR}
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
||||||
|
python ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
# Downgrade setuptools so that CKAN requirements can be built
|
||||||
|
RUN pip install setuptools==44.1.0
|
||||||
|
|
||||||
|
# Fetch and build the default CKAN extensions
|
||||||
|
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
|
||||||
|
|
||||||
|
############
|
||||||
|
### MAIN ###
|
||||||
|
############
|
||||||
|
FROM ubuntu:focal-20210827
|
||||||
|
|
||||||
|
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
||||||
|
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
|
||||||
|
|
||||||
|
# Set timezone
|
||||||
|
ENV TZ=UTC
|
||||||
|
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||||
|
|
||||||
|
# Set Locale
|
||||||
|
ENV LC_ALL=en_US.UTF-8
|
||||||
|
|
||||||
|
# Set the locale
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install --no-install-recommends -y locales && \
|
||||||
|
sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \
|
||||||
|
dpkg-reconfigure --frontend=noninteractive locales && \
|
||||||
|
update-locale LANG=${LC_ALL} && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV APP_DIR=/srv/app
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV CKAN_DIR=${SRC_DIR}/ckan
|
||||||
|
ENV DATA_DIR=/srv/app/data
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
ENV CKAN_SITE_URL=http://localhost:5000
|
||||||
|
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
|
||||||
|
|
||||||
|
# Install necessary packages to run CKAN
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install --no-install-recommends -y \
|
||||||
|
gettext \
|
||||||
|
curl \
|
||||||
|
ca-certificates \
|
||||||
|
libpq5 \
|
||||||
|
git \
|
||||||
|
postgresql-client \
|
||||||
|
python3 \
|
||||||
|
python3-distutils \
|
||||||
|
libpython3.8 \
|
||||||
|
libmagic1 \
|
||||||
|
libpcre3 \
|
||||||
|
libxslt1.1 \
|
||||||
|
libxml2 \
|
||||||
|
tzdata \
|
||||||
|
apache2-utils && \
|
||||||
|
rm -rf /var/lib/apt/lists/* && \
|
||||||
|
# Create SRC_DIR
|
||||||
|
mkdir -p ${SRC_DIR} && \
|
||||||
|
# Link python to python3
|
||||||
|
ln -s /usr/bin/python3 /usr/bin/python
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
||||||
|
python ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
# Downgrade setuptools so that CKAN requirements can be built
|
||||||
|
RUN pip install setuptools==44.1.0
|
||||||
|
|
||||||
|
# Get artifacts from build stages
|
||||||
|
COPY --from=ckanbuild /wheels /srv/app/wheels
|
||||||
|
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
||||||
|
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
|
||||||
|
|
||||||
|
# Additional install steps for build stages artifacts
|
||||||
|
RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2
|
||||||
|
|
||||||
|
# Create a local user and group to run the app
|
||||||
|
RUN groupadd -g 92 ckan && \
|
||||||
|
useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan
|
||||||
|
|
||||||
|
WORKDIR ${CKAN_DIR}
|
||||||
|
|
||||||
|
# Install CKAN
|
||||||
|
RUN pip install -e /srv/app/src/ckan && \
|
||||||
|
cp who.ini ${APP_DIR} && \
|
||||||
|
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
|
||||||
|
# Install default CKAN extensions
|
||||||
|
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
|
||||||
|
# Create and update CKAN config
|
||||||
|
# Generate CKAN config
|
||||||
|
ckan generate config ${APP_DIR}/production.ini && \
|
||||||
|
# Configure plugins
|
||||||
|
ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
||||||
|
# Create the data directory
|
||||||
|
mkdir ${DATA_DIR} && \
|
||||||
|
# Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created
|
||||||
|
ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \
|
||||||
|
# Set the default level for extensions to INFO
|
||||||
|
ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
|
||||||
|
# Change ownership to app user
|
||||||
|
chown -R ckan:ckan /srv/app
|
||||||
|
|
||||||
|
# Remove wheels
|
||||||
|
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
|
||||||
|
|
||||||
|
# Copy necessary scripts
|
||||||
|
COPY setup/app ${APP_DIR}
|
||||||
|
|
||||||
|
WORKDIR ${APP_DIR}
|
||||||
|
|
||||||
|
# Create entrypoint directory for children image scripts
|
||||||
|
ONBUILD RUN mkdir docker-entrypoint.d
|
||||||
|
|
||||||
|
# Create afterinit directory for children image scripts
|
||||||
|
ONBUILD RUN mkdir docker-afterinit.d
|
||||||
|
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
||||||
|
|
||||||
|
USER ckan
|
||||||
|
|
||||||
|
CMD ["/srv/app/start_ckan.sh"]
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
#!/bin/bash
|
||||||
|
shopt -s nullglob
|
||||||
|
for patch in patches/*.patch; do
|
||||||
|
/usr/bin/patch -p0 -i $patch
|
||||||
|
done
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# this is called before uwsgi is executed
|
||||||
|
# uset his to add extra scripts before ckan is started
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
<!--
|
||||||
|
Copyright (c) 2016 Keitaro AB
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Maintenance</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Maintenance</h1>
|
||||||
|
<p>Our data portal is currently in maintenance, please try in a while.</p>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
<!--
|
||||||
|
Copyright (c) 2016 Keitaro AB
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Maintenance</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Maintenance</h1>
|
||||||
|
<p>Our data portal is currently in maintenance, please try in a while.</p>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
"""
|
||||||
|
Copyright (c) 2016 Keitaro AB
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
|
||||||
|
import os
|
||||||
|
|
||||||
|
PORT = 5000
|
||||||
|
|
||||||
|
web_dir = os.path.join(os.path.dirname(__file__))
|
||||||
|
os.chdir(web_dir)
|
||||||
|
|
||||||
|
|
||||||
|
def run(server_class=ThreadingHTTPServer, handler_class=SimpleHTTPRequestHandler):
|
||||||
|
server_address = ("0.0.0.0", PORT)
|
||||||
|
httpd = server_class(server_address, handler_class)
|
||||||
|
print("Starting maintenance mode")
|
||||||
|
httpd.serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run()
|
||||||
|
|
||||||
|
|
@ -0,0 +1,228 @@
|
||||||
|
"""
|
||||||
|
Copyright (c) 2016 Keitaro AB
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
"""
|
||||||
|
from multiprocessing import connection
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import psycopg2
|
||||||
|
from sqlalchemy.engine.url import make_url
|
||||||
|
import urllib.request, urllib.error, urllib.parse, base64
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
|
||||||
|
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
|
||||||
|
|
||||||
|
RETRY = 5
|
||||||
|
|
||||||
|
def check_db_connection(retry=None):
|
||||||
|
|
||||||
|
print('[prerun] Start check_db_connection...')
|
||||||
|
|
||||||
|
if retry is None:
|
||||||
|
retry = RETRY
|
||||||
|
elif retry == 0:
|
||||||
|
print('[prerun] Giving up after 5 tries...')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
|
||||||
|
try:
|
||||||
|
db_user = make_url(conn_str).username
|
||||||
|
db_passwd = make_url(conn_str).password
|
||||||
|
db_host = make_url(conn_str).host
|
||||||
|
db_name = make_url(conn_str).database
|
||||||
|
connection = psycopg2.connect(user=db_user,
|
||||||
|
host=db_host,
|
||||||
|
password=db_passwd,
|
||||||
|
database=db_name)
|
||||||
|
|
||||||
|
except psycopg2.Error as e:
|
||||||
|
print((str(e)))
|
||||||
|
print('[prerun] Unable to connect to the database...try again in a while.')
|
||||||
|
import time
|
||||||
|
time.sleep(10)
|
||||||
|
check_db_connection(retry = retry - 1)
|
||||||
|
else:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
def check_solr_connection(retry=None):
|
||||||
|
|
||||||
|
print('[prerun] Start check_solr_connection...')
|
||||||
|
|
||||||
|
if retry is None:
|
||||||
|
retry = RETRY
|
||||||
|
elif retry == 0:
|
||||||
|
print('[prerun] Giving up after 5 tries...')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
url = os.environ.get('CKAN_SOLR_URL', '')
|
||||||
|
username = os.environ.get('CKAN_SOLR_USER', '')
|
||||||
|
password = os.environ.get('CKAN_SOLR_PASSWORD', '')
|
||||||
|
search_url = '{url}/schema/name?wt=json'.format(url=url)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if not username:
|
||||||
|
connection = urllib.request.urlopen(search_url)
|
||||||
|
else:
|
||||||
|
request = urllib.request.Request(search_url)
|
||||||
|
base64string = base64.b64encode(bytes('%s:%s' % (username, password),'ascii'))
|
||||||
|
request.add_header("Authorization", "Basic %s" % base64string.decode('utf-8'))
|
||||||
|
connection = urllib.request.urlopen(request)
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
print('[prerun] Unable to connect to solr...try again in a while.')
|
||||||
|
import time
|
||||||
|
time.sleep(10)
|
||||||
|
check_solr_connection(retry = retry - 1)
|
||||||
|
else:
|
||||||
|
import re
|
||||||
|
conn_info = connection.read()
|
||||||
|
schema_name = json.loads(conn_info)
|
||||||
|
if 'ckan' in schema_name['name']:
|
||||||
|
print('[prerun] Succesfully connected to solr and CKAN schema loaded')
|
||||||
|
else:
|
||||||
|
print('[prerun] Succesfully connected to solr, but CKAN schema not found')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
def init_db():
|
||||||
|
|
||||||
|
print('[prerun] Start init_db...')
|
||||||
|
|
||||||
|
db_command = ['ckan', '-c', ckan_ini, 'db', 'init']
|
||||||
|
|
||||||
|
print('[prerun] Initializing or upgrading db - start using ckan db init')
|
||||||
|
try:
|
||||||
|
# run init scripts
|
||||||
|
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
||||||
|
|
||||||
|
print('[prerun] Initializing or upgrading db - end')
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
if 'OperationalError' in str(e.output):
|
||||||
|
print(e.output.decode('utf-8'))
|
||||||
|
print('[prerun] Database not ready, waiting a bit before exit...')
|
||||||
|
import time
|
||||||
|
time.sleep(5)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print(e.output.decode('utf-8'))
|
||||||
|
raise e
|
||||||
|
print('[prerun] Initializing or upgrading db - finish')
|
||||||
|
|
||||||
|
|
||||||
|
def init_datastore():
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print('[prerun] Skipping datastore initialization')
|
||||||
|
return
|
||||||
|
|
||||||
|
datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore',
|
||||||
|
'set-permissions']
|
||||||
|
|
||||||
|
db_user = make_url(conn_str).username
|
||||||
|
db_passwd = make_url(conn_str).password
|
||||||
|
db_host = make_url(conn_str).host
|
||||||
|
db_name = make_url(conn_str).database
|
||||||
|
connection = psycopg2.connect(user=db_user,
|
||||||
|
host=db_host,
|
||||||
|
password=db_passwd,
|
||||||
|
database=db_name)
|
||||||
|
cursor = connection.cursor()
|
||||||
|
|
||||||
|
print('[prerun] Initializing datastore db - start')
|
||||||
|
try:
|
||||||
|
datastore_perms = subprocess.Popen(
|
||||||
|
datastore_perms_command,
|
||||||
|
stdout=subprocess.PIPE)
|
||||||
|
|
||||||
|
perms_sql = datastore_perms.stdout.read()
|
||||||
|
perms_sql = perms_sql.decode('utf-8')
|
||||||
|
perms_sql = perms_sql.replace("@"+db_host, "")
|
||||||
|
# Remove internal pg command as psycopg2 does not like it
|
||||||
|
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
||||||
|
cursor.execute(perms_sql)
|
||||||
|
for notice in connection.notices:
|
||||||
|
print(notice)
|
||||||
|
|
||||||
|
connection.commit()
|
||||||
|
|
||||||
|
print('[prerun] Initializing datastore db - end')
|
||||||
|
print((datastore_perms.stdout.read()))
|
||||||
|
except psycopg2.Error as e:
|
||||||
|
print('[prerun] Could not initialize datastore')
|
||||||
|
print(e.decode('utf-8'))
|
||||||
|
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
if 'OperationalError' in str(e.output):
|
||||||
|
print(e.output.decode('utf-8'))
|
||||||
|
print('[prerun] Database not ready, waiting a bit before exit...')
|
||||||
|
time.sleep(5)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print(e.output.decode('utf-8'))
|
||||||
|
raise e
|
||||||
|
finally:
|
||||||
|
cursor.close()
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def create_sysadmin():
|
||||||
|
|
||||||
|
print('[prerun] Start create_sysadmin...')
|
||||||
|
|
||||||
|
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
||||||
|
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
||||||
|
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
||||||
|
|
||||||
|
if name and password and email:
|
||||||
|
|
||||||
|
# Check if user exists
|
||||||
|
command = ['ckan', '-c', ckan_ini, 'user', 'show', name]
|
||||||
|
|
||||||
|
out = subprocess.check_output(command)
|
||||||
|
if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')):
|
||||||
|
print('[prerun] Sysadmin user exists, skipping creation')
|
||||||
|
return
|
||||||
|
|
||||||
|
# Create user
|
||||||
|
command = ['ckan', '-c', ckan_ini, 'user', 'add',
|
||||||
|
name,
|
||||||
|
'password=' + password,
|
||||||
|
'email=' + email]
|
||||||
|
|
||||||
|
subprocess.call(command)
|
||||||
|
print(('[prerun] Created user {0}'.format(name)))
|
||||||
|
|
||||||
|
# Make it sysadmin
|
||||||
|
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
|
||||||
|
name]
|
||||||
|
|
||||||
|
subprocess.call(command)
|
||||||
|
print(('[prerun] Made user {0} a sysadmin'.format(name)))
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
||||||
|
|
||||||
|
if maintenance:
|
||||||
|
print('[prerun] Maintenance mode, skipping setup...')
|
||||||
|
else:
|
||||||
|
check_db_connection()
|
||||||
|
check_solr_connection()
|
||||||
|
init_db()
|
||||||
|
if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
|
||||||
|
init_datastore()
|
||||||
|
create_sysadmin()
|
||||||
|
|
@ -0,0 +1,85 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
|
||||||
|
ckan config-tool $APP_DIR/production.ini ckan.datapusher.api_token=xxx
|
||||||
|
|
||||||
|
# Run any startup scripts provided by images extending this one
|
||||||
|
if [[ -d "${APP_DIR}/docker-entrypoint.d" ]]
|
||||||
|
then
|
||||||
|
for f in ${APP_DIR}/docker-entrypoint.d/*; do
|
||||||
|
case "$f" in
|
||||||
|
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
||||||
|
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
||||||
|
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
||||||
|
esac
|
||||||
|
echo
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Add session secret from chart
|
||||||
|
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET || -z $JWT_ENCODE_SECRET || -v $JWT_ENCODE_SECRET || -z $JWT_DECODE_SECRET || -v $JWT_DECODE_SECRET ]];then
|
||||||
|
echo "Not all environment variables are set. Generating sessions..."
|
||||||
|
else
|
||||||
|
echo "Setting session secrets from environment variables"
|
||||||
|
ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
|
||||||
|
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_ENCODE_SECRET"
|
||||||
|
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_DECODE_SECRET"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
|
||||||
|
then
|
||||||
|
echo "Setting secrets in ini file"
|
||||||
|
ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
|
||||||
|
ckan config-tool $APP_DIR/production.ini "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
|
||||||
|
JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
|
||||||
|
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_SECRET"
|
||||||
|
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_SECRET"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
|
||||||
|
UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid ckan --gid ckan --http :5000 --master --enable-threads --wsgi-file /srv/app/wsgi.py --module wsgi:application --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch --vacuum --harakiri 50 --callable application"
|
||||||
|
|
||||||
|
# Run the prerun script to init CKAN and create the default admin user
|
||||||
|
python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }
|
||||||
|
|
||||||
|
echo "Set up ckan.datapusher.api_token in the CKAN config file"
|
||||||
|
ckan config-tool $APP_DIR/production.ini "ckan.datapusher.api_token=$(ckan -c $APP_DIR/production.ini user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')"
|
||||||
|
|
||||||
|
# Check if we are in maintenance mode and if yes serve the maintenance pages
|
||||||
|
if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi
|
||||||
|
|
||||||
|
# Run any after prerun/init scripts provided by images extending this one
|
||||||
|
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
|
||||||
|
then
|
||||||
|
for f in ${APP_DIR}/docker-afterinit.d/*; do
|
||||||
|
case "$f" in
|
||||||
|
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
|
||||||
|
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
|
||||||
|
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
||||||
|
esac
|
||||||
|
echo
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
||||||
|
if [ $? -eq 0 ]
|
||||||
|
then
|
||||||
|
if [ "$PASSWORD_PROTECT" = true ]
|
||||||
|
then
|
||||||
|
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
||||||
|
then
|
||||||
|
# Generate htpasswd file for basicauth
|
||||||
|
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
||||||
|
# Start uwsgi with basicauth
|
||||||
|
uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
|
||||||
|
else
|
||||||
|
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Start uwsgi
|
||||||
|
uwsgi $UWSGI_OPTS
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "[prerun] failed...not starting CKAN."
|
||||||
|
fi
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
[uwsgi]
|
||||||
|
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
"""
|
||||||
|
Copyright (c) 2016 Keitaro AB
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import os
|
||||||
|
from ckan.config.middleware import make_app
|
||||||
|
from ckan.cli import CKANConfigLoader
|
||||||
|
from logging.config import fileConfig as loggingFileConfig
|
||||||
|
config_filepath = os.path.join(
|
||||||
|
os.path.dirname(os.path.abspath(__file__)), u'production.ini')
|
||||||
|
abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
loggingFileConfig(config_filepath)
|
||||||
|
config = CKANConfigLoader(config_filepath).get_config()
|
||||||
|
application = make_app(config)
|
||||||
Loading…
Reference in New Issue