diff --git a/ckan/setup/prerun.py.override b/ckan/setup/prerun.py.override new file mode 100644 index 0000000..3d68696 --- /dev/null +++ b/ckan/setup/prerun.py.override @@ -0,0 +1,212 @@ +import os +import sys +import subprocess +import psycopg2 +try: + from urllib.request import urlopen + from urllib.error import URLError +except ImportError: + from urllib2 import urlopen + from urllib2 import URLError + +import time +import re +import json + +ckan_ini = os.environ.get("CKAN_INI", "/srv/app/ckan.ini") + +RETRY = 5 + + +def update_plugins(): + + plugins = os.environ.get("CKAN__PLUGINS", "") + print(("[prerun] Setting the following plugins in {}:".format(ckan_ini))) + print(plugins) + cmd = ["ckan", "config-tool", ckan_ini, "ckan.plugins = {}".format(plugins)] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + print("[prerun] Plugins set.") + + +def check_main_db_connection(retry=None): + + conn_str = os.environ.get("CKAN_SQLALCHEMY_URL") + if not conn_str: + print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db") + return check_db_connection(conn_str, retry) + + +def check_datastore_db_connection(retry=None): + + conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") + if not conn_str: + print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db") + return check_db_connection(conn_str, retry) + + +def check_db_connection(conn_str, retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print("[prerun] Giving up after 5 tries...") + sys.exit(1) + + try: + connection = psycopg2.connect(conn_str) + + except psycopg2.Error as e: + print(str(e)) + print("[prerun] Unable to connect to the database, waiting...") + time.sleep(10) + check_db_connection(conn_str, retry=retry - 1) + else: + connection.close() + + +def check_solr_connection(retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print("[prerun] Giving up after 5 tries...") + sys.exit(1) + + url = os.environ.get("CKAN_SOLR_URL", "") + search_url = '{url}/schema/name?wt=json'.format(url=url) + + try: + connection = urlopen(search_url) + except URLError as e: + print(str(e)) + print("[prerun] Unable to connect to solr, waiting...") + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + import re + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[prerun] Succesfully connected to solr and CKAN schema loaded') + else: + print('[prerun] Succesfully connected to solr, but CKAN schema not found') + + +def init_db(): + + db_command = ["ckan", "-c", ckan_ini, "db", "init"] + print("[prerun] Initializing or upgrading db - start") + try: + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + print("[prerun] Initializing or upgrading db - end") + except subprocess.CalledProcessError as e: + if "OperationalError" in e.output: + print(e.output) + print("[prerun] Database not ready, waiting a bit before exit...") + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + + +def init_datastore_db(): + + conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") + if not conn_str: + print("[prerun] Skipping datastore initialization") + return + + datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"] + + connection = psycopg2.connect(conn_str) + cursor = connection.cursor() + + print("[prerun] Initializing datastore db - start") + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, stdout=subprocess.PIPE + ) + + perms_sql = datastore_perms.stdout.read() + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print(notice) + + connection.commit() + + print("[prerun] Initializing datastore db - end") + print(datastore_perms.stdout.read()) + except psycopg2.Error as e: + print("[prerun] Could not initialize datastore") + print(str(e)) + + except subprocess.CalledProcessError as e: + if "OperationalError" in e.output: + print(e.output) + print("[prerun] Database not ready, waiting a bit before exit...") + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + name = os.environ.get("CKAN_SYSADMIN_NAME") + password = os.environ.get("CKAN_SYSADMIN_PASSWORD") + email = os.environ.get("CKAN_SYSADMIN_EMAIL") + + if name and password and email: + + # Check if user exists + command = ["ckan", "-c", ckan_ini, "user", "show", name] + + out = subprocess.check_output(command) + if b"User:None" not in re.sub(b"\s", b"", out): + print("[prerun] Sysadmin user exists, skipping creation") + return + + # Create user + command = [ + "ckan", + "-c", + ckan_ini, + "user", + "add", + name, + "password=" + password, + "email=" + email, + ] + + subprocess.call(command) + print("[prerun] Created user {0}".format(name)) + + # Make it sysadmin + command = ["ckan", "-c", ckan_ini, "sysadmin", "add", name] + + subprocess.call(command) + print("[prerun] Made user {0} a sysadmin".format(name)) + + +if __name__ == "__main__": + + maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true" + + if maintenance: + print("[prerun] Maintenance mode, skipping setup...") + else: + check_main_db_connection() + init_db() + update_plugins() + check_datastore_db_connection() + init_datastore_db() + check_solr_connection() + create_sysadmin() + \ No newline at end of file diff --git a/ckan/setup/prerun.py.override.override b/ckan/setup/prerun.py.override.override index b148d78..e69de29 100644 --- a/ckan/setup/prerun.py.override.override +++ b/ckan/setup/prerun.py.override.override @@ -1,211 +0,0 @@ -import os -import sys -import subprocess -import psycopg2 -try: - from urllib.request import urlopen - from urllib.error import URLError -except ImportError: - from urllib2 import urlopen - from urllib2 import URLError - -import time -import re -import json - -ckan_ini = os.environ.get("CKAN_INI", "/srv/app/ckan.ini") - -RETRY = 5 - - -def update_plugins(): - - plugins = os.environ.get("CKAN__PLUGINS", "") - print(("[prerun] Setting the following plugins in {}:".format(ckan_ini))) - print(plugins) - cmd = ["ckan", "config-tool", ckan_ini, "ckan.plugins = {}".format(plugins)] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - print("[prerun] Plugins set.") - - -def check_main_db_connection(retry=None): - - conn_str = os.environ.get("CKAN_SQLALCHEMY_URL") - if not conn_str: - print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db") - return check_db_connection(conn_str, retry) - - -def check_datastore_db_connection(retry=None): - - conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") - if not conn_str: - print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db") - return check_db_connection(conn_str, retry) - - -def check_db_connection(conn_str, retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print("[prerun] Giving up after 5 tries...") - sys.exit(1) - - try: - connection = psycopg2.connect(conn_str) - - except psycopg2.Error as e: - print(str(e)) - print("[prerun] Unable to connect to the database, waiting...") - time.sleep(10) - check_db_connection(conn_str, retry=retry - 1) - else: - connection.close() - - -def check_solr_connection(retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print("[prerun] Giving up after 5 tries...") - sys.exit(1) - - url = os.environ.get("CKAN_SOLR_URL", "") - search_url = '{url}/schema/name?wt=json'.format(url=url) - - try: - connection = urlopen(search_url) - except URLError as e: - print(str(e)) - print("[prerun] Unable to connect to solr, waiting...") - time.sleep(10) - check_solr_connection(retry=retry - 1) - else: - import re - conn_info = connection.read() - schema_name = json.loads(conn_info) - if 'ckan' in schema_name['name']: - print('[prerun] Succesfully connected to solr and CKAN schema loaded') - else: - print('[prerun] Succesfully connected to solr, but CKAN schema not found') - - -def init_db(): - - db_command = ["ckan", "-c", ckan_ini, "db", "init"] - print("[prerun] Initializing or upgrading db - start") - try: - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print("[prerun] Initializing or upgrading db - end") - except subprocess.CalledProcessError as e: - if "OperationalError" in e.output: - print(e.output) - print("[prerun] Database not ready, waiting a bit before exit...") - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - - -def init_datastore_db(): - - conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") - if not conn_str: - print("[prerun] Skipping datastore initialization") - return - - datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"] - - connection = psycopg2.connect(conn_str) - cursor = connection.cursor() - - print("[prerun] Initializing datastore db - start") - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, stdout=subprocess.PIPE - ) - - perms_sql = datastore_perms.stdout.read() - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print(notice) - - connection.commit() - - print("[prerun] Initializing datastore db - end") - print(datastore_perms.stdout.read()) - except psycopg2.Error as e: - print("[prerun] Could not initialize datastore") - print(str(e)) - - except subprocess.CalledProcessError as e: - if "OperationalError" in e.output: - print(e.output) - print("[prerun] Database not ready, waiting a bit before exit...") - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - name = os.environ.get("CKAN_SYSADMIN_NAME") - password = os.environ.get("CKAN_SYSADMIN_PASSWORD") - email = os.environ.get("CKAN_SYSADMIN_EMAIL") - - if name and password and email: - - # Check if user exists - command = ["ckan", "-c", ckan_ini, "user", "show", name] - - out = subprocess.check_output(command) - if b"User:None" not in re.sub(b"\s", b"", out): - print("[prerun] Sysadmin user exists, skipping creation") - return - - # Create user - command = [ - "ckan", - "-c", - ckan_ini, - "user", - "add", - name, - "password=" + password, - "email=" + email, - ] - - subprocess.call(command) - print("[prerun] Created user {0}".format(name)) - - # Make it sysadmin - command = ["ckan", "-c", ckan_ini, "sysadmin", "add", name] - - subprocess.call(command) - print("[prerun] Made user {0} a sysadmin".format(name)) - - -if __name__ == "__main__": - - maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true" - - if maintenance: - print("[prerun] Maintenance mode, skipping setup...") - else: - check_main_db_connection() - init_db() - update_plugins() - check_datastore_db_connection() - init_datastore_db() - check_solr_connection() - create_sysadmin() diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index ce6eebd..b9d5de0 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -18,6 +18,9 @@ fi # Run the prerun script to init CKAN and create the default admin user sudo -u ckan -EH python3 prerun.py +echo "Set up ckan.datapusher.api_token in the CKAN config file" +ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" + # Run any startup scripts provided by images extending this one if [[ -d "/docker-entrypoint.d" ]] then diff --git a/ckan/setup/supervisord.conf b/ckan/setup/supervisord.conf deleted file mode 100644 index a3f6671..0000000 --- a/ckan/setup/supervisord.conf +++ /dev/null @@ -1,23 +0,0 @@ -[unix_http_server] -file = /tmp/supervisor.sock -chmod = 0777 -chown = nobody:nogroup - -[supervisord] -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = info -pidfile = /tmp/supervisord.pid -nodaemon = true -umask = 022 -identifier = supervisor - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[include] -files = /etc/supervisord.d/*.conf diff --git a/ckan/setup/uwsgi.conf b/ckan/setup/uwsgi.conf deleted file mode 100644 index 6321d6d..0000000 --- a/ckan/setup/uwsgi.conf +++ /dev/null @@ -1,2 +0,0 @@ -[uwsgi] -route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd