Merge pull request #44 from mjanez/fix/dev-mode

Fix development mode
2023-05-03 09:25:54 +02:00 · 2023-05-03 09:25:54 +02:00 · 0858b397ec
parent 0e727962bd 65c038afd0
commit 0858b397ec
9 changed files with 199 additions and 32 deletions
--- a/.gitignore
+++ b/.gitignore
@ -13,6 +13,8 @@ log/*
 !log/README.md
 metadata/*
 !metadata/README.md
+src/*
+!src/README.md

 # environment
 .env
--- a/README.md
+++ b/README.md
@ -331,10 +331,30 @@ The Docker image config files used to build your CKAN project are located in the
 * `Dockerfile`: this is based on `ckan/ckan-base-spatial:<version>`, a base image located in the [Github Package Registry](https://github.com/mjanez/ckan-docker/pkgs/container/ckan-base-spatial), that has CKAN installed along with all its dependencies, properly configured and running on [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/) (production setup)
 * `Dockerfile.dev`:  this is based on `ckan/ckan-base-spatial:<version>-dev` also located located in the Github Package Registry, and extends `ckan/ckan-base-spatial:<version>` to include:

-  * Any extension cloned on the `src` folder will be installed in the CKAN container when booting up Docker Compose (`docker compose up`). This includes installing any requirements listed in a `requirements.txt` (or `pip-requirements.txt`) file and running `python setup.py develop`.
+  * Any extension cloned on the `./src` folder will be installed in the CKAN container when booting up Docker Compose (`docker compose up`). This includes installing any requirements listed in a `requirements.txt` (or `pip-requirements.txt`) file and running `python setup.py develop`.
  * CKAN is started running this: `/usr/bin/ckan -c /srv/app/ckan.ini run -H 0.0.0.0`.
  * Make sure to add the local plugins to the `CKAN__PLUGINS` env var in the `.env` file.

+* Any custom changes to the scripts run during container start up can be made to scripts in the `setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, and the `start_ckan.sh.override` file. Then you would need to add the following line to the Dockerfile ie: `COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh`. The `start_ckan.sh` file in the locally built image would override the `start_ckan.sh` file included in the base image
+
+>**Note**<br>
+> If you get an error like ` doesn't have execute permissions`: 
+>
+>```log
+>Daemon error response: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: exec: "/srv/app/start_ckan.sh": permission denied: unknown
+>```
+>
+>It may be necessary to give execute permissions to the file in the `Dockerfile`:
+>
+>```dockerfile
+>...
+># Override start_ckan.sh
+>COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh
+>RUN chmod +x ${APP_DIR}/start_ckan.sh
+>...
+>```
+
+

 ## CKAN images enhancement
 ### Extending the base images
--- a/ckan/Dockerfile
+++ b/ckan/Dockerfile
@ -1,14 +1,8 @@
 FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.8

 # Set up environment variables
-ENV APP_DIR=/srv/app
-ENV TZ=UTC
-RUN echo ${TZ} > /etc/timezone
-
-# Make sure both files are not exactly the same
-RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
-    cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
-    fi ;
+ENV APP_DIR=/srv/app \
+    TZ=UTC

 # Extensions
 ### XLoader - v0.12.2 ###
@ -20,7 +14,10 @@ RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
 ### Resource dictionary - 1.0.1 ###
 ### Pages - v0.5.1 ###
 ### PDFView - v0.0.8 ###
-RUN echo "ckan/ckanext-xloader" && \
+RUN echo ${TZ} > /etc/timezone && \
+    if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime ; fi && \
+    # Install CKAN extensions
+    echo "ckan/ckanext-xloader" && \
    pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@0.12.2#egg=ckanext-xloader' && \ 
    pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \
    pip3 install -U requests[security] && \
--- a/ckan/Dockerfile.dev
+++ b/ckan/Dockerfile.dev
@ -1,14 +1,13 @@
-FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.8
+FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.8-dev

 # Set up environment variables
-ENV APP_DIR=/srv/app
-ENV TZ=UTC
-RUN echo ${TZ} > /etc/timezone
+ENV APP_DIR=/srv/app \
+    TZ=UTC \
+    SRC_EXTENSIONS_DIR=/srv/app/src_extensions

+RUN echo ${TZ} > /etc/timezone && \
    # Make sure both files are not exactly the same
-RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
-    cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
-    fi ;
+    if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime ; fi

 # Install any extensions needed by your CKAN instance
 # - Make sure to add the plugins to CKAN__PLUGINS in the .env file
@ -52,14 +51,18 @@ COPY docker-entrypoint.d/* /docker-entrypoint.d/
 # Update who.ini with APACHE_CKAN_LOCATION
 COPY setup/who.ini ${APP_DIR}/

+# Override start_ckan.sh with DEV sh
+COPY setup/start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh
+RUN chmod +x ${APP_DIR}/start_ckan_development.sh
+
 # Apply any patches needed to CKAN core or any of the built extensions (not the
 # runtime mounted ones)
-COPY patches ${APP_DIR}/patches
+# COPY patches ${APP_DIR}/patches

-RUN for d in $APP_DIR/patches/*; do \
-    if [ -d $d ]; then \
-    for f in `ls $d/*.patch | sort -g`; do \
-    cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
-    done ; \
-    fi ; \
-    done
+# RUN for d in $APP_DIR/patches/*; do \
+#     if [ -d $d ]; then \
+#     for f in `ls $d/*.patch | sort -g`; do \
+#     cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+#     done ; \
+#     fi ; \
+#     done
--- a/ckan/setup/prerun.py.override.override
+++ b/ckan/setup/prerun.py.override.override
--- a/ckan/setup/start_ckan.sh.override
+++ b/ckan/setup/start_ckan.sh.override
@ -1,4 +1,7 @@
-#!/bin/bash
+#!/bin/sh
+
+# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
+ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx

 # Set up the Secret key used by Beaker and Flask
 # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
@ -6,6 +9,7 @@ if grep -E "beaker.session.secret ?= ?$" ckan.ini
 then
    echo "Setting beaker.session.secret in ini file"
    ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
    JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
    ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}"
    ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
--- a/ckan/setup/start_ckan_development.sh.override
+++ b/ckan/setup/start_ckan_development.sh.override
@ -0,0 +1,96 @@
+#!/bin/sh
+
+# Install any local extensions in the src_extensions volume
+echo "Looking for local extensions to install..."
+echo "Extension dir contents:"
+ls -la $SRC_EXTENSIONS_DIR
+for i in $SRC_EXTENSIONS_DIR/*
+do
+    if [ -d $i ];
+    then
+
+        if [ -f $i/pip-requirements.txt ];
+        then
+            pip install -r $i/pip-requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/requirements.txt ];
+        then
+            pip install -r $i/requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/dev-requirements.txt ];
+        then
+            pip install -r $i/dev-requirements.txt
+            echo "Found dev-requirements file in $i"
+        fi
+        if [ -f $i/setup.py ];
+        then
+            cd $i
+            python3 $i/setup.py develop
+            echo "Found setup.py file in $i"
+            cd $APP_DIR
+        fi
+
+        # Point `use` in test.ini to location of `test-core.ini`
+        if [ -f $i/test.ini ];
+        then
+            echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i"
+            ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini"
+        fi
+    fi
+done
+
+# Set debug to true
+echo "Enabling debug mode"
+ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
+
+# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
+ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx
+
+# Set up the Secret key used by Beaker and Flask
+# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
+if grep -E "beaker.session.secret ?= ?$" ckan.ini
+then
+    echo "Setting beaker.session.secret in ini file"
+    ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
+    ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}"
+    ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
+fi
+
+# Update the plugins setting in the ini file with the values defined in the env var
+echo "Loading the following plugins: $CKAN__PLUGINS"
+ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
+
+# Update test-core.ini DB, SOLR & Redis settings
+echo "Loading test settings into test-core.ini"
+ckan config-tool $SRC_DIR/ckan/test-core.ini \
+    "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \
+    "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \
+    "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \
+    "solr_url = $TEST_CKAN_SOLR_URL" \
+    "ckan.redis.url = $TEST_CKAN_REDIS_URL"
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python3 prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python3 "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Start supervisord
+supervisord --configuration /etc/supervisord.conf &
+
+# Start the development server with automatic reload
+sudo -u ckan -EH ckan -c $CKAN_INI run -H 0.0.0.0
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -7,6 +7,20 @@ volumes:

 services:
   
+  apache:
+    container_name: ${APACHE_CONTAINER_NAME}
+    build:
+      context: apache/
+      dockerfile: Dockerfile
+    env_file:
+      - .env
+    depends_on:
+      ckan-dev:
+        condition: service_healthy
+    ports:
+      - "0.0.0.0:${APACHE_PORT_HOST}:${APACHE_PORT}"
+    restart: on-failure:3
+  
  ckan-dev:
    container_name: ${CKAN_CONTAINER_NAME}
    build:
@ -35,7 +49,26 @@ services:
      - ./src:/srv/app/src_extensions
    restart: unless-stopped
    healthcheck:
-      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"]
+
+  pycsw:
+    container_name: ${PYCSW_CONTAINER_NAME}
+    build:
+      context: ckan-pycsw/
+      dockerfile: Dockerfile
+    env_file:
+      - .env
+    depends_on:
+      ckan-dev:
+        condition: service_healthy
+    ports:
+      - "0.0.0.0:${PYCSW_PORT_HOST}:${PYCSW_PORT}"
+    volumes:
+        - ./log:${APP_DIR}/log
+        - ./metadata:${APP_DIR}/metadata
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"]
    
  db:
    container_name: ${POSTGRESQL_CONTAINER_NAME}
@ -61,7 +94,11 @@ services:
     
  solr:
    container_name: ${SOLR_CONTAINER_NAME}
-    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    build:
+      context: solr/
+      dockerfile: Dockerfile.spatial
+    env_file:
+      - .env
    logging:
      driver: "json-file"
      options:
@ -75,6 +112,9 @@ services:

  redis:
    container_name: ${REDIS_CONTAINER_NAME}
+    # build:
+    #   context: redis/
+    #   dockerfile: Dockerfile
    image: redis:${REDIS_VERSION}
    logging:
      driver: "json-file"
--- a/src/README.md
+++ b/src/README.md
@ -0,0 +1,5 @@
+# Development SRC folder
+This folder is used to clone CKAN extensions in development mode.
+
+>**Warning**
+> Save the extension code in this folder before updating/deleting the repo, all contents (`src/*)` are in `.gitignore` file.