More commits
This commit is contained in:
Brett
parent
1eb9630aa5
commit
089bcf590c
|
|
@ -1,3 +1,12 @@
|
|||
# Container names
|
||||
NGINX_CONTAINER_NAME=nginx
|
||||
REDIS_CONTAINER_NAME=redis
|
||||
POSTGRESQL_CONTAINER_NAME=db
|
||||
SOLR_CONTAINER_NAME=solr
|
||||
DATAPUSHER_CONTAINER_NAME=datapusher
|
||||
CKAN_CONTAINER_NAME=ckan
|
||||
WORKER_CONTAINER_NAME=ckan-worker
|
||||
|
||||
# CKAN databases
|
||||
POSTGRES_USER=ckan
|
||||
POSTGRES_PASSWORD=ckan
|
||||
|
|
@ -14,6 +23,7 @@ TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
|
|||
TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
|
||||
|
||||
# CKAN core
|
||||
CKAN_VERSION=2.9.5
|
||||
CKAN_SITE_ID=default
|
||||
CKAN_SITE_URL=http://ckan:5000
|
||||
CKAN_PORT=5000
|
||||
|
|
@ -29,12 +39,12 @@ CKAN_SMTP_MAIL_FROM=ckan@localhost
|
|||
TZ=UTC
|
||||
|
||||
# Solr
|
||||
SOLR_VERSION=6.6.6
|
||||
SOLR_IMAGE_VERSION=2.9-solr8
|
||||
CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
||||
TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
||||
|
||||
# Redis
|
||||
REDIS_VERSION=6.0.7
|
||||
REDIS_VERSION=6
|
||||
CKAN_REDIS_URL=redis://redis:6379/1
|
||||
TEST_CKAN_REDIS_URL=redis://redis:6379/1
|
||||
|
||||
|
|
@ -46,7 +56,7 @@ DATAPUSHER_REWRITE_RESOURCES=True
|
|||
DATAPUSHER_REWRITE_URL=http://ckan:5000
|
||||
|
||||
# Extensions
|
||||
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
|
||||
CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher"
|
||||
CKAN__HARVEST__MQ__TYPE=redis
|
||||
CKAN__HARVEST__MQ__HOSTNAME=redis
|
||||
CKAN__HARVEST__MQ__PORT=6379
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ All the other images should live in separate repos
|
|||
latest image to used is redis:6
|
||||
4. nginx - base image: nginx:1.19.8-alpine from DockerHub, enhanced in a Dockerfile)
|
||||
5. DataPusher - built from the actual datapusher repo (https://github.com/ckan/datapusher)
|
||||
6. CKAN Worker - add new (ckan worker) container in the compose setup
|
||||
6. CKAN - built from the ckan/ckan-base:2.9.5 base image (which is built from the ckan/ckan-docker-base repo)
|
||||
7. CKAN Worker - add new (ckan worker) container in the compose setup
|
||||
|
||||
Versions 2.9 and 2.10 (when it's out) only. Plan the repo layout for having multiple versions - OKFN could used as an example
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,204 @@
|
|||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import psycopg2
|
||||
try:
|
||||
from urllib.request import urlopen
|
||||
from urllib.error import URLError
|
||||
except ImportError:
|
||||
from urllib2 import urlopen
|
||||
from urllib2 import URLError
|
||||
|
||||
import time
|
||||
import re
|
||||
|
||||
ckan_ini = os.environ.get("CKAN_INI", "/srv/app/ckan.ini")
|
||||
|
||||
RETRY = 5
|
||||
|
||||
|
||||
def update_plugins():
|
||||
|
||||
plugins = os.environ.get("CKAN__PLUGINS", "")
|
||||
print(("[prerun] Setting the following plugins in {}:".format(ckan_ini)))
|
||||
print(plugins)
|
||||
cmd = ["ckan", "config-tool", ckan_ini, "ckan.plugins = {}".format(plugins)]
|
||||
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||
print("[prerun] Plugins set.")
|
||||
|
||||
|
||||
def check_main_db_connection(retry=None):
|
||||
|
||||
conn_str = os.environ.get("CKAN_SQLALCHEMY_URL")
|
||||
if not conn_str:
|
||||
print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db")
|
||||
return check_db_connection(conn_str, retry)
|
||||
|
||||
|
||||
def check_datastore_db_connection(retry=None):
|
||||
|
||||
conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL")
|
||||
if not conn_str:
|
||||
print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db")
|
||||
return check_db_connection(conn_str, retry)
|
||||
|
||||
|
||||
def check_db_connection(conn_str, retry=None):
|
||||
|
||||
if retry is None:
|
||||
retry = RETRY
|
||||
elif retry == 0:
|
||||
print("[prerun] Giving up after 5 tries...")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
connection = psycopg2.connect(conn_str)
|
||||
|
||||
except psycopg2.Error as e:
|
||||
print(str(e))
|
||||
print("[prerun] Unable to connect to the database, waiting...")
|
||||
time.sleep(10)
|
||||
check_db_connection(conn_str, retry=retry - 1)
|
||||
else:
|
||||
connection.close()
|
||||
|
||||
|
||||
def check_solr_connection(retry=None):
|
||||
|
||||
if retry is None:
|
||||
retry = RETRY
|
||||
elif retry == 0:
|
||||
print("[prerun] Giving up after 5 tries...")
|
||||
sys.exit(1)
|
||||
|
||||
url = os.environ.get("CKAN_SOLR_URL", "")
|
||||
search_url = "{url}/select/?q=*&wt=json".format(url=url)
|
||||
|
||||
try:
|
||||
connection = urlopen(search_url)
|
||||
except URLError as e:
|
||||
print(str(e))
|
||||
print("[prerun] Unable to connect to solr, waiting...")
|
||||
time.sleep(10)
|
||||
check_solr_connection(retry=retry - 1)
|
||||
else:
|
||||
eval(connection.read())
|
||||
|
||||
|
||||
def init_db():
|
||||
|
||||
db_command = ["ckan", "-c", ckan_ini, "db", "init"]
|
||||
print("[prerun] Initializing or upgrading db - start")
|
||||
try:
|
||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
||||
print("[prerun] Initializing or upgrading db - end")
|
||||
except subprocess.CalledProcessError as e:
|
||||
if "OperationalError" in e.output:
|
||||
print(e.output)
|
||||
print("[prerun] Database not ready, waiting a bit before exit...")
|
||||
time.sleep(5)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(e.output)
|
||||
raise e
|
||||
|
||||
|
||||
def init_datastore_db():
|
||||
|
||||
conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL")
|
||||
if not conn_str:
|
||||
print("[prerun] Skipping datastore initialization")
|
||||
return
|
||||
|
||||
datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"]
|
||||
|
||||
connection = psycopg2.connect(conn_str)
|
||||
cursor = connection.cursor()
|
||||
|
||||
print("[prerun] Initializing datastore db - start")
|
||||
try:
|
||||
datastore_perms = subprocess.Popen(
|
||||
datastore_perms_command, stdout=subprocess.PIPE
|
||||
)
|
||||
|
||||
perms_sql = datastore_perms.stdout.read()
|
||||
# Remove internal pg command as psycopg2 does not like it
|
||||
perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql)
|
||||
cursor.execute(perms_sql)
|
||||
for notice in connection.notices:
|
||||
print(notice)
|
||||
|
||||
connection.commit()
|
||||
|
||||
print("[prerun] Initializing datastore db - end")
|
||||
print(datastore_perms.stdout.read())
|
||||
except psycopg2.Error as e:
|
||||
print("[prerun] Could not initialize datastore")
|
||||
print(str(e))
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
if "OperationalError" in e.output:
|
||||
print(e.output)
|
||||
print("[prerun] Database not ready, waiting a bit before exit...")
|
||||
time.sleep(5)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(e.output)
|
||||
raise e
|
||||
finally:
|
||||
cursor.close()
|
||||
connection.close()
|
||||
|
||||
|
||||
def create_sysadmin():
|
||||
|
||||
name = os.environ.get("CKAN_SYSADMIN_NAME")
|
||||
password = os.environ.get("CKAN_SYSADMIN_PASSWORD")
|
||||
email = os.environ.get("CKAN_SYSADMIN_EMAIL")
|
||||
|
||||
if name and password and email:
|
||||
|
||||
# Check if user exists
|
||||
command = ["ckan", "-c", ckan_ini, "user", "show", name]
|
||||
|
||||
out = subprocess.check_output(command)
|
||||
if b"User:None" not in re.sub(b"\s", b"", out):
|
||||
print("[prerun] Sysadmin user exists, skipping creation")
|
||||
return
|
||||
|
||||
# Create user
|
||||
command = [
|
||||
"ckan",
|
||||
"-c",
|
||||
ckan_ini,
|
||||
"user",
|
||||
"add",
|
||||
name,
|
||||
"password=" + password,
|
||||
"email=" + email,
|
||||
]
|
||||
|
||||
subprocess.call(command)
|
||||
print("[prerun] Created user {0}".format(name))
|
||||
|
||||
# Make it sysadmin
|
||||
command = ["ckan", "-c", ckan_ini, "sysadmin", "add", name]
|
||||
|
||||
subprocess.call(command)
|
||||
print("[prerun] Made user {0} a sysadmin".format(name))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true"
|
||||
|
||||
if maintenance:
|
||||
print("[prerun] Maintenance mode, skipping setup...")
|
||||
else:
|
||||
check_main_db_connection()
|
||||
init_db()
|
||||
update_plugins()
|
||||
check_datastore_db_connection()
|
||||
init_datastore_db()
|
||||
check_solr_connection()
|
||||
create_sysadmin()
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Run the prerun script to init CKAN and create the default admin user
|
||||
sudo -u ckan -EH python3 prerun.py
|
||||
|
||||
# Run any startup scripts provided by images extending this one
|
||||
if [[ -d "/docker-entrypoint.d" ]]
|
||||
then
|
||||
for f in /docker-entrypoint.d/*; do
|
||||
case "$f" in
|
||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
||||
*.py) echo "$0: Running init file $f"; python3 "$f"; echo ;;
|
||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
||||
esac
|
||||
echo
|
||||
done
|
||||
fi
|
||||
|
||||
# Set the common uwsgi options
|
||||
UWSGI_OPTS="--plugins http,python \
|
||||
--socket /tmp/uwsgi.sock \
|
||||
--wsgi-file /srv/app/wsgi.py \
|
||||
--module wsgi:application \
|
||||
--uid 92 --gid 92 \
|
||||
--http 0.0.0.0:5000 \
|
||||
--master --enable-threads \
|
||||
--lazy-apps \
|
||||
-p 2 -L -b 32768 --vacuum \
|
||||
--harakiri $UWSGI_HARAKIRI"
|
||||
|
||||
if [ $? -eq 0 ]
|
||||
then
|
||||
# Start supervisord
|
||||
supervisord --configuration /etc/supervisord.conf &
|
||||
# Start uwsgi
|
||||
sudo -u ckan -EH uwsgi $UWSGI_OPTS
|
||||
else
|
||||
echo "[prerun] failed...not starting CKAN."
|
||||
fi
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
[unix_http_server]
|
||||
file = /tmp/supervisor.sock
|
||||
chmod = 0777
|
||||
chown = nobody:nogroup
|
||||
|
||||
[supervisord]
|
||||
logfile = /tmp/supervisord.log
|
||||
logfile_maxbytes = 50MB
|
||||
logfile_backups=10
|
||||
loglevel = info
|
||||
pidfile = /tmp/supervisord.pid
|
||||
nodaemon = true
|
||||
umask = 022
|
||||
identifier = supervisor
|
||||
|
||||
[supervisorctl]
|
||||
serverurl = unix:///tmp/supervisor.sock
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
[include]
|
||||
files = /etc/supervisord.d/*.conf
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
[uwsgi]
|
||||
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
||||
|
|
@ -26,6 +26,8 @@ services:
|
|||
dockerfile: Dockerfile
|
||||
args:
|
||||
- TZ=${TZ}
|
||||
env_file:
|
||||
- .env
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
Loading…
Reference in New Issue