From 0ba0561f3189c8a946cb2431dfffac4fcd8c730c Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Tue, 2 May 2023 14:47:10 +0200
Subject: [PATCH 1/2] Fix development mode

---
 .gitignore                                    |  2 +
 README.md                                     |  4 +-
 ckan/Dockerfile                               | 15 ++-
 ckan/Dockerfile.dev                           | 18 ++--
 ...{prerun.py => prerun.py.override.override} |  2 +-
 .../{start_ckan.sh => start_ckan.sh.override} |  8 +-
 ckan/setup/start_ckan_development.sh.override | 96 +++++++++++++++++++
 docker-compose.dev.yml                        | 46 ++++++++-
 src/README.md                                 |  5 +
 9 files changed, 172 insertions(+), 24 deletions(-)
 rename ckan/setup/{prerun.py => prerun.py.override.override} (99%)
 rename ckan/setup/{start_ckan.sh => start_ckan.sh.override} (86%)
 mode change 100755 => 100644
 create mode 100644 ckan/setup/start_ckan_development.sh.override
 create mode 100644 src/README.md

diff --git a/.gitignore b/.gitignore
index 2c7c45a..b125056 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,8 @@ log/*
 !log/README.md
 metadata/*
 !metadata/README.md
+src/*
+!src/README.md
 
 # environment
 .env
\ No newline at end of file
diff --git a/README.md b/README.md
index 94884d3..dbd915c 100644
--- a/README.md
+++ b/README.md
@@ -331,10 +331,12 @@ The Docker image config files used to build your CKAN project are located in the
 * `Dockerfile`: this is based on `ckan/ckan-base-spatial:<version>`, a base image located in the [Github Package Registry](https://github.com/mjanez/ckan-docker/pkgs/container/ckan-base-spatial), that has CKAN installed along with all its dependencies, properly configured and running on [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/) (production setup)
 * `Dockerfile.dev`:  this is based on `ckan/ckan-base-spatial:<version>-dev` also located located in the Github Package Registry, and extends `ckan/ckan-base-spatial:<version>` to include:
 
-  * Any extension cloned on the `src` folder will be installed in the CKAN container when booting up Docker Compose (`docker compose up`). This includes installing any requirements listed in a `requirements.txt` (or `pip-requirements.txt`) file and running `python setup.py develop`.
+  * Any extension cloned on the `./src` folder will be installed in the CKAN container when booting up Docker Compose (`docker compose up`). This includes installing any requirements listed in a `requirements.txt` (or `pip-requirements.txt`) file and running `python setup.py develop`.
   * CKAN is started running this: `/usr/bin/ckan -c /srv/app/ckan.ini run -H 0.0.0.0`.
   * Make sure to add the local plugins to the `CKAN__PLUGINS` env var in the `.env` file.
 
+* Any custom changes to the scripts run during container start up can be made to scripts in the `setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, and the `start_ckan.sh.override` file. Then you would need to add the following line to the Dockerfile ie: `COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh`. The `start_ckan.sh` file in the locally built image would override the `start_ckan.sh` file included in the base image
+
 
 ## CKAN images enhancement
 ### Extending the base images
diff --git a/ckan/Dockerfile b/ckan/Dockerfile
index 2df4635..84e2a2d 100644
--- a/ckan/Dockerfile
+++ b/ckan/Dockerfile
@@ -1,14 +1,8 @@
 FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.8
 
 # Set up environment variables
-ENV APP_DIR=/srv/app
-ENV TZ=UTC
-RUN echo ${TZ} > /etc/timezone
-
-# Make sure both files are not exactly the same
-RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
-    cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
-    fi ;
+ENV APP_DIR=/srv/app \
+    TZ=UTC
 
 # Extensions
 ### XLoader - v0.12.2 ###
@@ -20,7 +14,10 @@ RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
 ### Resource dictionary - 1.0.1 ###
 ### Pages - v0.5.1 ###
 ### PDFView - v0.0.8 ###
-RUN echo "ckan/ckanext-xloader" && \
+RUN echo ${TZ} > /etc/timezone && \
+    if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime ; fi && \
+    # Install CKAN extensions
+    echo "ckan/ckanext-xloader" && \
     pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@0.12.2#egg=ckanext-xloader' && \ 
     pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \
     pip3 install -U requests[security] && \
diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev
index c92d5a3..c1f5927 100644
--- a/ckan/Dockerfile.dev
+++ b/ckan/Dockerfile.dev
@@ -1,14 +1,13 @@
-FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.8
+FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.8-dev
 
 # Set up environment variables
-ENV APP_DIR=/srv/app
-ENV TZ=UTC
-RUN echo ${TZ} > /etc/timezone
+ENV APP_DIR=/srv/app \
+    TZ=UTC \
+    SRC_EXTENSIONS_DIR=/srv/app/src_extensions
 
-# Make sure both files are not exactly the same
-RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
-    cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
-    fi ;
+RUN echo ${TZ} > /etc/timezone && \
+    # Make sure both files are not exactly the same
+    if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime ; fi
 
 # Install any extensions needed by your CKAN instance
 # - Make sure to add the plugins to CKAN__PLUGINS in the .env file
@@ -52,6 +51,9 @@ COPY docker-entrypoint.d/* /docker-entrypoint.d/
 # Update who.ini with APACHE_CKAN_LOCATION
 COPY setup/who.ini ${APP_DIR}/
 
+# Override start_ckan.sh with DEV sh
+# COPY setup/start_ckan_development.sh.override ${APP_DIR}/start_ckan.sh
+
 # Apply any patches needed to CKAN core or any of the built extensions (not the
 # runtime mounted ones)
 COPY patches ${APP_DIR}/patches
diff --git a/ckan/setup/prerun.py b/ckan/setup/prerun.py.override.override
similarity index 99%
rename from ckan/setup/prerun.py
rename to ckan/setup/prerun.py.override.override
index b148d78..368f796 100644
--- a/ckan/setup/prerun.py
+++ b/ckan/setup/prerun.py.override.override
@@ -208,4 +208,4 @@ if __name__ == "__main__":
         check_datastore_db_connection()
         init_datastore_db()
         check_solr_connection()
-        create_sysadmin()
+        create_sysadmin()
\ No newline at end of file
diff --git a/ckan/setup/start_ckan.sh b/ckan/setup/start_ckan.sh.override
old mode 100755
new mode 100644
similarity index 86%
rename from ckan/setup/start_ckan.sh
rename to ckan/setup/start_ckan.sh.override
index d6d437d..ce6eebd
--- a/ckan/setup/start_ckan.sh
+++ b/ckan/setup/start_ckan.sh.override
@@ -1,4 +1,7 @@
-#!/bin/bash
+#!/bin/sh
+
+# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
+ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx
 
 # Set up the Secret key used by Beaker and Flask
 # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
@@ -6,6 +9,7 @@ if grep -E "beaker.session.secret ?= ?$" ckan.ini
 then
     echo "Setting beaker.session.secret in ini file"
     ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
     JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
     ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}"
     ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
@@ -47,4 +51,4 @@ then
     sudo -u ckan -EH uwsgi $UWSGI_OPTS
 else
   echo "[prerun] failed...not starting CKAN."
-fi
+fi
\ No newline at end of file
diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override
new file mode 100644
index 0000000..8dcc746
--- /dev/null
+++ b/ckan/setup/start_ckan_development.sh.override
@@ -0,0 +1,96 @@
+#!/bin/sh
+
+# Install any local extensions in the src_extensions volume
+echo "Looking for local extensions to install..."
+echo "Extension dir contents:"
+ls -la $SRC_EXTENSIONS_DIR
+for i in $SRC_EXTENSIONS_DIR/*
+do
+    if [ -d $i ];
+    then
+
+        if [ -f $i/pip-requirements.txt ];
+        then
+            pip install -r $i/pip-requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/requirements.txt ];
+        then
+            pip install -r $i/requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/dev-requirements.txt ];
+        then
+            pip install -r $i/dev-requirements.txt
+            echo "Found dev-requirements file in $i"
+        fi
+        if [ -f $i/setup.py ];
+        then
+            cd $i
+            python3 $i/setup.py develop
+            echo "Found setup.py file in $i"
+            cd $APP_DIR
+        fi
+
+        # Point `use` in test.ini to location of `test-core.ini`
+        if [ -f $i/test.ini ];
+        then
+            echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i"
+            ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini"
+        fi
+    fi
+done
+
+# Set debug to true
+echo "Enabling debug mode"
+ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
+
+# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
+ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx
+
+# Set up the Secret key used by Beaker and Flask
+# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
+if grep -E "beaker.session.secret ?= ?$" ckan.ini
+then
+    echo "Setting beaker.session.secret in ini file"
+    ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
+    ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}"
+    ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
+fi
+
+# Update the plugins setting in the ini file with the values defined in the env var
+echo "Loading the following plugins: $CKAN__PLUGINS"
+ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
+
+# Update test-core.ini DB, SOLR & Redis settings
+echo "Loading test settings into test-core.ini"
+ckan config-tool $SRC_DIR/ckan/test-core.ini \
+    "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \
+    "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \
+    "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \
+    "solr_url = $TEST_CKAN_SOLR_URL" \
+    "ckan.redis.url = $TEST_CKAN_REDIS_URL"
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python3 prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python3 "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Start supervisord
+supervisord --configuration /etc/supervisord.conf &
+
+# Start the development server with automatic reload
+sudo -u ckan -EH ckan -c $CKAN_INI run -H 0.0.0.0
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index e0b462e..3492e61 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -6,7 +6,21 @@ volumes:
   solr_data:
 
 services:
-
+   
+  apache:
+    container_name: ${APACHE_CONTAINER_NAME}
+    build:
+      context: apache/
+      dockerfile: Dockerfile
+    env_file:
+      - .env
+    depends_on:
+      ckan-dev:
+        condition: service_healthy
+    ports:
+      - "0.0.0.0:${APACHE_PORT_HOST}:${APACHE_PORT}"
+    restart: on-failure:3
+  
   ckan-dev:
     container_name: ${CKAN_CONTAINER_NAME}
     build:
@@ -35,7 +49,26 @@ services:
       - ./src:/srv/app/src_extensions
     restart: unless-stopped
     healthcheck:
-      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"]
+
+  pycsw:
+    container_name: ${PYCSW_CONTAINER_NAME}
+    build:
+      context: ckan-pycsw/
+      dockerfile: Dockerfile
+    env_file:
+      - .env
+    depends_on:
+      ckan-dev:
+        condition: service_healthy
+    ports:
+      - "0.0.0.0:${PYCSW_PORT_HOST}:${PYCSW_PORT}"
+    volumes:
+        - ./log:${APP_DIR}/log
+        - ./metadata:${APP_DIR}/metadata
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"]
     
   db:
     container_name: ${POSTGRESQL_CONTAINER_NAME}
@@ -61,7 +94,11 @@ services:
      
   solr:
     container_name: ${SOLR_CONTAINER_NAME}
-    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    build:
+      context: solr/
+      dockerfile: Dockerfile.spatial
+    env_file:
+      - .env
     logging:
       driver: "json-file"
       options:
@@ -75,6 +112,9 @@ services:
 
   redis:
     container_name: ${REDIS_CONTAINER_NAME}
+    # build:
+    #   context: redis/
+    #   dockerfile: Dockerfile
     image: redis:${REDIS_VERSION}
     logging:
       driver: "json-file"
diff --git a/src/README.md b/src/README.md
new file mode 100644
index 0000000..0a600b1
--- /dev/null
+++ b/src/README.md
@@ -0,0 +1,5 @@
+# Development SRC folder
+This folder is used to clone CKAN extensions in development mode.
+
+>**Warning**
+> Save the extension code in this folder before updating/deleting the repo, all contents (`src/*)` are in `.gitignore` file.
\ No newline at end of file

From 65c038afd0c118cca94672225c9c5fa1249f5948 Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Wed, 3 May 2023 07:24:42 +0000
Subject: [PATCH 2/2] Fix Dockerfile.dev and add info to doc

Avoid: Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: exec: "/srv/app/start_ckan_development.sh": permission denied: unknown
---
 README.md           | 18 ++++++++++++++++++
 ckan/Dockerfile.dev | 19 ++++++++++---------
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index dbd915c..7f88138 100644
--- a/README.md
+++ b/README.md
@@ -337,6 +337,24 @@ The Docker image config files used to build your CKAN project are located in the
 
 * Any custom changes to the scripts run during container start up can be made to scripts in the `setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, and the `start_ckan.sh.override` file. Then you would need to add the following line to the Dockerfile ie: `COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh`. The `start_ckan.sh` file in the locally built image would override the `start_ckan.sh` file included in the base image
 
+>**Note**<br>
+> If you get an error like ` doesn't have execute permissions`: 
+>
+>```log
+>Daemon error response: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: exec: "/srv/app/start_ckan.sh": permission denied: unknown
+>```
+>
+>It may be necessary to give execute permissions to the file in the `Dockerfile`:
+>
+>```dockerfile
+>...
+># Override start_ckan.sh
+>COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh
+>RUN chmod +x ${APP_DIR}/start_ckan.sh
+>...
+>```
+
+
 
 ## CKAN images enhancement
 ### Extending the base images
diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev
index c1f5927..2c853d9 100644
--- a/ckan/Dockerfile.dev
+++ b/ckan/Dockerfile.dev
@@ -52,16 +52,17 @@ COPY docker-entrypoint.d/* /docker-entrypoint.d/
 COPY setup/who.ini ${APP_DIR}/
 
 # Override start_ckan.sh with DEV sh
-# COPY setup/start_ckan_development.sh.override ${APP_DIR}/start_ckan.sh
+COPY setup/start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh
+RUN chmod +x ${APP_DIR}/start_ckan_development.sh
 
 # Apply any patches needed to CKAN core or any of the built extensions (not the
 # runtime mounted ones)
-COPY patches ${APP_DIR}/patches
+# COPY patches ${APP_DIR}/patches
 
-RUN for d in $APP_DIR/patches/*; do \
-    if [ -d $d ]; then \
-    for f in `ls $d/*.patch | sort -g`; do \
-    cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
-    done ; \
-    fi ; \
-    done
\ No newline at end of file
+# RUN for d in $APP_DIR/patches/*; do \
+#     if [ -d $d ]; then \
+#     for f in `ls $d/*.patch | sort -g`; do \
+#     cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+#     done ; \
+#     fi ; \
+#     done
\ No newline at end of file