More updates

ckan-dev/2.9/Dockerfile

@@ -0,0 +1,29 @@
+FROM kowhai/ckan-base:2.9
+
+LABEL maintainer="brett@kowh.ai"
+
+ENV APP_DIR=/srv/app
+ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
+
+# Install packages needed by the dev requirements
+RUN apk add --no-cache libffi-dev
+
+# Set up Python3 virtual environment
+RUN cd ${APP_DIR} && \
+    source ${APP_DIR}/bin/activate
+
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${APP_DIR}/bin:${PATH} 
+
+# Install CKAN dev requirements
+# Will need to change this eventually - when CKAN 2.9 is out
+# wget https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+# RUN pip3 install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+RUN pip3 install -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+
+# Create folder for local extensions sources
+RUN mkdir $SRC_EXTENSIONS_DIR
+
+COPY setup/start_ckan_development.sh ${APP_DIR}
+
+CMD ["/srv/app/start_ckan_development.sh"]

ckan-dev/README

@@ -0,0 +1,4 @@
+Build the images from this directory using:
+
+    docker build -t openknowledge/ckan-dev:2.7 -f 2.7/Dockerfile .
+    docker build -t openknowledge/ckan-dev:2.8 -f 2.8/Dockerfile .

ckan-dev/setup/start_ckan_development.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# Install any local extensions in the src_extensions volume
+echo "Looking for local extensions to install..."
+echo "Extension dir contents:"
+ls -la $SRC_EXTENSIONS_DIR
+for i in $SRC_EXTENSIONS_DIR/*
+do
+    if [ -d $i ];
+    then
+
+        if [ -f $i/pip-requirements.txt ];
+        then
+            pip install -r $i/pip-requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/requirements.txt ];
+        then
+            pip install -r $i/requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/dev-requirements.txt ];
+        then
+            pip install -r $i/dev-requirements.txt
+            echo "Found dev-requirements file in $i"
+        fi
+        if [ -f $i/setup.py ];
+        then
+            cd $i
+            python $i/setup.py develop
+            echo "Found setup.py file in $i"
+            cd $APP_DIR
+        fi
+
+        # Point `use` in test.ini to location of `test-core.ini`
+        if [ -f $i/test.ini ];
+        then
+            echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i"
+            paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini"
+        fi
+    fi
+done
+
+# Set debug to true
+echo "Enabling debug mode"
+ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
+
+# Update the plugins setting in the ini file with the values defined in the env var
+echo "Loading the following plugins: $CKAN__PLUGINS"
+ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
+
+# Update test-core.ini DB, SOLR & Redis settings
+echo "Loading test settings into test-core.ini"
+ckan config-tool $SRC_DIR/ckan/test-core.ini \
+    "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \
+    "ckan.datstore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \
+    "ckan.datstore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \
+    "solr_url = $TEST_CKAN_SOLR_URL" \
+    "ckan.redis_url = $TEST_CKAN_REDIS_URL"
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Start supervisord
+supervisord --configuration /etc/supervisord.conf &
+
+# Start the development server with automatic reload
+# Check the --reloader options sudo -u ckan -EH ckan -c $CKAN_INI run --reloader <TEXT>
+sudo -u ckan -EH ckan -c $CKAN_INI run

dev-requirements.txt

@@ -0,0 +1,25 @@
+# These are packages that are required by ckan developers - for running ckan in debug mode, running ckan tests, building the docs and to pip-compile the requirements.in file.
+
+beautifulsoup4==4.5.1
+coveralls   #Let Unpinned - Requires latest coveralls
+docutils==0.12
+factory-boy==2.1.1
+Flask-DebugToolbar==0.10.1
+freezegun==0.3.15
+responses==0.10.6
+mock==2.0.0
+pycodestyle==2.5.0
+pip-tools==2.0.2
+pyfakefs==3.2
+Sphinx==1.8.5
+sphinx-rtd-theme==0.3.1
+cookiecutter==1.6.0
+
+# nose==1.3.0  # already in requirements.txt
+pytest==4.6.5
+pytest-split-tests==1.0.9
+pytest-cov==2.7.1
+pytest-freezegun==0.4.1
+pytest-rerunfailures==8.0
+
+towncrier==19.2.0

docker-compose.dev.yml

@@ -0,0 +1,56 @@
+version: "3"
+
+services:
+  ckan-dev:
+    build:
+      context: ckan/
+      dockerfile: Dockerfile.dev
+      args:
+        - TZ=${TZ}
+    env_file:
+      - .env
+    links:
+      - db
+      - solr
+      - redis
+      - datapusher
+    ports:
+      - "0.0.0.0:${CKAN_PORT}:5000"
+    volumes:
+      - ./src:/srv/app/src_extensions
+      - ckan_storage:/var/lib/ckan
+    
+
+  datapusher:
+    container_name: datapusher
+    build:
+      context: datapusher/
+    ports:
+      - "8800:8800"
+
+  db:
+    container_name: db
+    env_file:
+      - .env
+    build:
+      context: postgresql/
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+
+  solr:
+    container_name: solr
+    build:
+      context: solr/
+    ports:
+      - "8983:8983"
+    volumes:
+      - solr_data:/opt/solr/server/solr/ckan/data/index
+
+  redis:
+    container_name: redis
+    image: redis:alpine
+
+volumes:
+  ckan_storage:
+  pg_data:
+  solr_data: