Merge pull request #68 from keitaroinc/solr8-support

Add SOLR8 in compose setup since CKAN 2.9.5 supports it

Readme.md

@@ -29,16 +29,16 @@ Directory layout:
 ## Running CKAN using docker-compose
 To start CKAN using docker-compose, simply change into the *compose* directory and run
 ```sh
-cd compose
+cd compose/2.9
 docker-compose build
 docker-compose up
 ```
 Check if CKAN was succesfuly started on http://localhost:5000. 
 
 ### Configuration
-In order to configure CKAN within docker-compose we use both build/up time variables loaded via the [.env](./compose/.env) file, and runtime variables loaded via the [.ckan-env](./compose/.ckan-env) file. 
+In order to configure CKAN within docker-compose we use both build/up time variables loaded via the [.env](./compose/2.9/.env) file, and runtime variables loaded via the [.ckan-env](./compose/2.9/.ckan-env) file. 
 
-Variables in the [.env](./compose/.env) file are loaded when running `docker-compose build` and `docker-compose up`, while variables in [.ckan-env](./compose/.ckan-env) file are used withing the CKAN container at runtime to configure CKAN and CKAN extensions using [ckanext-envvars](https://github.com/okfn/ckanext-envvars).
+Variables in the [.env](./compose/2.9/.env) file are loaded when running `docker-compose build` and `docker-compose up`, while variables in [.ckan-env](./compose/2.9/.ckan-env) file are used withing the CKAN container at runtime to configure CKAN and CKAN extensions using [ckanext-envvars](https://github.com/okfn/ckanext-envvars).
 
 ## Extending CKAN docker images
 Check some examples of extending CKAN docker images in the [examples](./examples) directory.

compose/2.8/.env

@@ -12,7 +12,7 @@ POSTGRES_PORT=5432
 DATASTORE_READONLY_PASSWORD=datastore
 
 # CKAN
-CKAN_VERSION=2.9.5
+CKAN_VERSION=2.8.10
 CKAN_SITE_ID=default
 CKAN_SITE_URL=http://localhost:5000
 CKAN_PORT=5000

compose/2.9/.ckan-env

@@ -0,0 +1,36 @@
+# Runtime configuration of CKAN enabled through ckanext-envvars
+# Information about how it works: https://github.com/okfn/ckanext-envvars
+# Note that variables here take presedence over build/up time variables in .env
+
+# Set to true to disable CKAN from starting and serve a maintenance page
+MAINTENANCE_MODE=false
+
+# General Settings
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5000
+CKAN_MAX_UPLOAD_SIZE_MB=10
+# CKAN Plugins
+CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
+# CKAN requires storage path to be set in order for filestore to be enabled
+CKAN__STORAGE_PATH=/srv/app/data
+CKAN__WEBASSETS__PATH=/srv/app/data/webassets
+# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
+CKAN_SYSADMIN_NAME=sysadmin
+CKAN_SYSADMIN_PASSWORD=password
+CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
+
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
+
+# Solr configuration
+CKAN_VERSION=2.9.5
+CKAN_CORE_NAME=ckan

compose/2.9/.env

@@ -0,0 +1,35 @@
+# Variables in this file will be used as build arguments when running
+# docker-compose build and docker-compose up
+# Verify correct substitution with "docker-compose config"
+# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
+# docker-compose down -v
+# docker-compose build
+# docker-compose up -d
+
+# Database
+POSTGRES_PASSWORD=ckan
+POSTGRES_PORT=5432
+DATASTORE_READONLY_PASSWORD=datastore
+
+# CKAN
+CKAN_VERSION=2.9.5
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5000
+CKAN_MAX_UPLOAD_SIZE_MB=10
+
+# Datapusher
+DATAPUSHER_VERSION=0.0.17
+DATAPUSHER_MAX_CONTENT_LENGTH=10485760
+DATAPUSHER_CHUNK_SIZE=16384
+DATAPUSHER_CHUNK_INSERT_ROWS=250
+DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000
+
+# SOLR
+CKAN_CORE_NAME=ckan
+
+# Redis
+REDIS_VERSION=6.0.7

compose/2.9/docker-compose.yml

@@ -0,0 +1,94 @@
+# docker-compose build && docker-compose up -d
+version: "3"
+
+volumes:
+  ckan_data:
+  pg_data:
+  solr_data:
+
+services:
+  ckan:
+    container_name: ckan
+    image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION}
+    networks:
+      - frontend
+      - backend
+    depends_on:
+      - db
+    ports:
+      - "0.0.0.0:${CKAN_PORT}:5000"
+    env_file:
+      - ./.ckan-env
+    environment:
+      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
+      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
+      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
+      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
+      - CKAN_REDIS_URL=redis://redis:6379/1
+      - CKAN_SITE_URL=${CKAN_SITE_URL}
+      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+    volumes:
+      - ckan_data:/srv/app/data
+
+  datapusher:
+    container_name: datapusher
+    image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
+    networks:
+      - frontend
+      - backend
+    ports:
+      - "8000:8000"
+    environment:
+      - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
+      - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
+      - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
+      - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
+      - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
+
+
+  db:
+    container_name: db
+    build:
+      context: .
+      dockerfile: postgresql/Dockerfile
+      args:
+        - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    networks:
+      - backend
+    environment:
+      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - PGDATA=/var/lib/postgresql/data/db
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "ckan"]
+
+  solr:
+    container_name: solr
+    image: solr:8.11.1
+    networks:
+      - backend
+    env_file:
+      - ./.ckan-env
+    environment:
+      - CKAN_CORE_NAME=${CKAN_CORE_NAME}
+      - CKAN_VERSION=${CKAN_VERSION}
+    volumes:
+      - solr_data:/var/solr
+      - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
+
+  redis:
+    container_name: redis
+    image: redis:${REDIS_VERSION}
+    networks:
+      - backend
+
+networks:
+  frontend:
+  backend:

compose/2.9/postgresql/Dockerfile

@@ -0,0 +1,13 @@
+FROM mdillon/postgis:11
+
+# Allow connections; we don't map out any ports so only linked docker containers can connect
+RUN echo "host all  all    0.0.0.0/0  md5" >> /var/lib/postgresql/data/pg_hba.conf
+
+# Customize default user/pass/db
+ENV POSTGRES_DB ckan
+ENV POSTGRES_USER ckan
+ARG POSTGRES_PASSWORD
+ARG DS_RO_PASS
+
+# Include datastore setup scripts
+COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d

compose/2.9/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+    CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
+    CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
+    GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;
+EOSQL

compose/2.9/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql

@@ -0,0 +1,3 @@
+CREATE EXTENSION POSTGIS;
+ALTER VIEW geometry_columns OWNER TO ckan;
+ALTER TABLE spatial_ref_sys OWNER TO ckan;

compose/2.9/solr8/ckan_init_solr.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Initialize SOLR for CKAN by creating a ckan core
+# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION
+# Example:
+#   CKAN_CORE_NAME=ckan
+#   CKAN_VERSION=2.9.5
+
+set -e
+
+CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.solr8.xml 
+
+echo "Check whether managed schema exists for CKAN $CKAN_VERSION"
+if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then
+  echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..."
+  exit 1
+fi
+
+echo "Check whether SOLR is initialized for CKAN"
+CORESDIR=/var/solr/data
+
+COREDIR="$CORESDIR/$CKAN_CORE_NAME"
+if [ -d "$COREDIR" ]; then
+    echo "SOLR already initialized, skipping initialization"
+else
+    echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION"
+    # init script for handling an empty /var/solr
+    /opt/docker-solr/scripts/init-var-solr
+    
+    # Precreate CKAN core
+    /opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME
+    
+    # Replace the managed schema with CKANs schema
+    echo "Adding CKAN managed schema"
+    curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s
+    
+    echo "SOLR initialized"
+fi

examples/harvest/.ckan-env

@@ -31,3 +31,7 @@ CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
 # Harvest settings
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
+
+# Solr configuration
+CKAN_VERSION=2.9.5
+CKAN_CORE_NAME=ckan

examples/harvest/.env

@@ -28,5 +28,8 @@ DATAPUSHER_SSL_VERIFY=False
 DATAPUSHER_REWRITE_RESOURCES=True
 DATAPUSHER_REWRITE_URL=http://ckan:5000
 
+# SOLR
+CKAN_CORE_NAME=ckan
+
 # Redis
 REDIS_VERSION=6.0.7

examples/harvest/Dockerfile

@@ -5,16 +5,14 @@ FROM ghcr.io/keitaroinc/ckan:2.9.5 as extbuild
 
 # Locations and tags, please use specific tags or revisions
 ENV HARVEST_GIT_URL=https://github.com/ckan/ckanext-harvest
-ENV HARVEST_GIT_BRANCH=v1.3.1
+ENV HARVEST_GIT_BRANCH=v1.3.4
 
 # Switch to the root user
 USER root
 
 # Install necessary packages to build extensions
-# Make sure we install python 3.8, cause CKAN is not compatible with 3.9
-RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/v3.13/main \
-        python3-dev=3.8.10-r0 && \
-    apk add --no-cache \
+RUN apk add --no-cache \
+        python3-dev \
         gcc \
         g++ \
         libffi-dev \

examples/harvest/docker-compose.yml

@@ -37,7 +37,7 @@ services:
     container_name: ckan-harvest-gather
     build:
       context: .
-    command: ckan -c /srv/app/production.ini harvester gather_consumer
+    command: ckan -c /srv/app/production.ini harvester gather-consumer
     restart: on-failure
     networks:
       - frontend
@@ -62,7 +62,7 @@ services:
     container_name: ckan-harvest-fetch
     build:
       context: .
-    command: ckan -c /srv/app/production.ini harvester fetch_consumer
+    command: ckan -c /srv/app/production.ini harvester fetch-consumer
     restart: on-failure
     networks:
       - frontend
@@ -151,7 +151,7 @@ services:
   db:
     container_name: db
     build:
-      context: ../../compose
+      context: ../../compose/2.9
       dockerfile: postgresql/Dockerfile
       args:
         - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
@@ -169,15 +169,17 @@ services:
 
   solr:
     container_name: solr
-    build:
-      context: ../../compose
-      dockerfile: solr/Dockerfile
-      args:
-        - CKAN_VERSION=${CKAN_VERSION}
+    image: solr:8.11.1
     networks:
       - backend
+    env_file:
+      - ./.ckan-env
+    environment:
+      - CKAN_CORE_NAME=${CKAN_CORE_NAME}
+      - CKAN_VERSION=${CKAN_VERSION}
     volumes:
-      - solr_data:/opt/solr/server/solr/ckan/data
+      - solr_data:/var/solr
+      - ${PWD}/../../compose/2.9/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
 
   redis:
     container_name: redis

examples/s3filestore/.ckan-env

@@ -36,3 +36,7 @@ CKANEXT__S3FILESTORE__AWS_BUCKET_NAME=ckan
 CKANEXT__S3FILESTORE__HOST_NAME=http://minio:9000
 CKANEXT__S3FILESTORE__REGION_NAME=us-east-1
 CKANEXT__S3FILESTORE__SIGNATURE_VERSION=s3v4
+
+# Solr configuration
+CKAN_VERSION=2.9.5
+CKAN_CORE_NAME=ckan

examples/s3filestore/.env

@@ -28,5 +28,8 @@ DATAPUSHER_SSL_VERIFY=False
 DATAPUSHER_REWRITE_RESOURCES=True
 DATAPUSHER_REWRITE_URL=http://ckan:5000
 
+# SOLR
+CKAN_CORE_NAME=ckan
+
 # Redis
 REDIS_VERSION=6.0.7

examples/s3filestore/Dockerfile

@@ -5,7 +5,7 @@ FROM ghcr.io/keitaroinc/ckan:2.9.5 as extbuild
 
 # Locations and tags, please use specific tags or revisions
 ENV S3FILESTORE_GIT_URL=https://github.com/keitaroinc/ckanext-s3filestore
-ENV S3FILESTORE_GIT_BRANCH=master
+ENV S3FILESTORE_GIT_BRANCH=v1.0.0
 
 # Switch to the root user
 USER root

examples/s3filestore/docker-compose.yml

@@ -51,7 +51,7 @@ services:
   db:
     container_name: db
     build:
-      context: ../../compose
+      context: ../../compose/2.9
       dockerfile: postgresql/Dockerfile
       args:
         - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
@@ -69,15 +69,17 @@ services:
 
   solr:
     container_name: solr
-    build:
-      context: ../../compose
-      dockerfile: solr/Dockerfile
-      args:
-        - CKAN_VERSION=${CKAN_VERSION}
+    image: solr:8.11.1
     networks:
       - backend
+    env_file:
+      - ./.ckan-env
+    environment:
+      - CKAN_CORE_NAME=${CKAN_CORE_NAME}
+      - CKAN_VERSION=${CKAN_VERSION}
     volumes:
-      - solr_data:/opt/solr/server/solr/ckan/data
+      - solr_data:/var/solr
+      - ${PWD}/../../compose/2.9/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
 
   redis:
     container_name: redis

images/ckan/2.7/Dockerfile

@@ -113,6 +113,7 @@ ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
 RUN apk add --no-cache \ 
         git \
         bash \
+        git \
         gettext \
         curl \
         python2 \

images/ckan/2.8/Dockerfile

@@ -108,6 +108,7 @@ ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
 # Install necessary packages to run CKAN
 RUN apk add --no-cache git \
         bash \
+        git \
         gettext \
         curl \
         postgresql-client \

images/ckan/2.9/Dockerfile

@@ -118,6 +118,7 @@ ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
 RUN apk add --no-cache \
         python3 \
         bash \
+        git \
         gettext \
         curl \
         postgresql-client \

images/ckan/2.9/setup/app/prerun.py

@@ -21,6 +21,7 @@ import psycopg2
 from sqlalchemy.engine.url import make_url
 import urllib.request, urllib.error, urllib.parse
 import re
+import json
 
 import time
 
@@ -69,7 +70,7 @@ def check_solr_connection(retry=None):
         sys.exit(1)
 
     url = os.environ.get('CKAN_SOLR_URL', '')
-    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
+    search_url = '{url}/schema/name?wt=json'.format(url=url)
 
     try:
         connection = urllib.request.urlopen(search_url)
@@ -81,9 +82,12 @@ def check_solr_connection(retry=None):
     else:
         import re
         conn_info = connection.read()
-        # SolrCloud
-        conn_info = re.sub(r'"zkConnected":true', '"zkConnected":True', conn_info.decode('utf-8'))
-        eval(conn_info)
+        schema_name = json.loads(conn_info)
+        if 'ckan' in schema_name['name']:
+            print('[prerun] Succesfully connected to solr and CKAN schema loaded')
+        else:
+            print('[prerun] Succesfully connected to solr, but CKAN schema not found')
+            sys.exit(1)
            
 def init_db():