feat: Migrated to CKAN 2.11

Makefile

@@ -1,14 +1,14 @@
-VERSION := 2.10.4-focal
+VERSION := 2.11.0-focal
 DOCKERHUB_NAMESPACE ?= d4science
 IMAGE := ${DOCKERHUB_NAMESPACE}/ckan:${VERSION}
 
 build:
-	DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build --tag ${IMAGE} images/ckan/2.10
+	DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build -f images/ckan/2.11/Dockerfile.focal --tag ${IMAGE} images/ckan/2.11
 
 up:
-	cd compose/2.10 && docker compose --env-file .docker.env up -d
+	cd compose/2.11 && docker compose up -d
 up_build:
-	cd compose/2.10 && docker compose --env-file .docker.env up --build -d
+	cd compose/2.11 && docker compose up --build -d
 
 push: build
 	docker push ${IMAGE}

compose/2.10/.ckan.env

@@ -1,44 +0,0 @@
-# Runtime configuration of CKAN enabled through ckanext-envvars
-# Information about how it works: https://github.com/okfn/ckanext-envvars
-# Note that variables here take presedence over build/up time variables in .env
-
-DEVELOPMENT=true
-
-# Set to true to disable CKAN from starting and serve a maintenance page
-MAINTENANCE_MODE=false
-
-# General Settings
-CKAN_SITE_ID=default
-CKAN_SITE_URL=http://localhost:5000
-CKAN_PORT=5000
-CKAN_MAX_UPLOAD_SIZE_MB=200
-CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
-# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
-CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
-CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
-# CKAN Plugins
-CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher spatial_metadata spatial_query d4science d4science_theme"
-# CKAN requires storage path to be set in order for filestore to be enabled
-CKAN__STORAGE_PATH=/srv/app/data
-CKAN__WEBASSETS__PATH=/srv/app/data/webassets
-# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
-CKAN_SYSADMIN_NAME=sysadmin
-CKAN_SYSADMIN_PASSWORD=password
-CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
-
-# Email settings
-CKAN_SMTP_SERVER=smtp.corporateict.domain:25
-CKAN_SMTP_STARTTLS=True
-CKAN_SMTP_USER=user
-CKAN_SMTP_PASSWORD=pass
-CKAN_SMTP_MAIL_FROM=ckan@localhost
-
-# Datapusher configuration
-CKAN__DATAPUSHER__URL=http://datapusher:8000
-CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
-CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0
-# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts
-
-# Solr configuration
-CKAN_VERSION=2.10
-CKAN_CORE_NAME=ckan

compose/2.10/.docker.env

@@ -1,36 +0,0 @@
-# Variables in this file will be used as build arguments when running
-# docker-compose build and docker-compose up
-# Verify correct substitution with "docker-compose config"
-# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
-# docker-compose down -v
-# docker-compose build
-# docker-compose up -d
-
-# Database
-POSTGRES_PASSWORD=ckan
-POSTGRES_PORT=5432
-DATASTORE_READONLY_PASSWORD=datastore
-
-# CKAN
-CKAN_VERSION=2.10.5
-CKAN_SITE_ID=default
-CKAN_SITE_URL=http://localhost:5000
-CKAN_PORT=5000
-CKAN_MAX_UPLOAD_SIZE_MB=10
-
-# Datapusher
-DATAPUSHER_VERSION=0.0.17
-DATAPUSHER_MAX_CONTENT_LENGTH=10485760
-DATAPUSHER_CHUNK_SIZE=16384
-DATAPUSHER_CHUNK_INSERT_ROWS=250
-DATAPUSHER_DOWNLOAD_TIMEOUT=30
-DATAPUSHER_SSL_VERIFY=False
-DATAPUSHER_REWRITE_RESOURCES=True
-DATAPUSHER_REWRITE_URL=http://ckan:5005
-DATAPUSHER_SSL_VERIFY=False
-
-# SOLR
-CKAN_CORE_NAME=ckan
-
-# Redis
-REDIS_VERSION=6.0.7

compose/2.10/ckan-docker.code-workspace

@@ -1,8 +0,0 @@
-{
-	"folders": [
-		{
-			"path": "../.."
-		}
-	],
-	"settings": {}
-}

compose/2.10/docker-compose.yml

@@ -1,116 +0,0 @@
-volumes:
-  ckan_data:
-  pg_data:
-  solr_data:
-
-services:
-  ckan:
-    platform: linux/amd64
-    container_name: ckan
-    build:
-      context: .
-    networks:
-      - frontend
-      - backend
-    depends_on:
-      db:
-        condition: service_healthy
-        restart: true
-      solr:
-        condition: service_healthy
-        restart: true
-    ports:
-      - "0.0.0.0:${CKAN_PORT}:5000"
-    env_file:
-      - ./.ckan.env
-    environment:
-      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
-      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
-      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
-      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
-      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_SITE_URL=${CKAN_SITE_URL}
-      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
-    volumes:
-      - ckan_data:/srv/app/data
-      - ./src:/srv/app/src_extensions
-      - ./docker-entrypoint.d:/srv/app/docker-entrypoint.d
-
-  datapusher:
-    platform: linux/amd64
-    container_name: datapusher
-    image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
-    networks:
-      - frontend
-      - backend
-    ports:
-      - "8000:8000"
-    environment:
-      - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
-      - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
-      - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
-      - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
-      - DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
-      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
-      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
-
-
-  db:
-    platform: linux/amd64
-    container_name: db
-    build:
-      context: .
-      dockerfile: postgresql/Dockerfile
-      args:
-        - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
-        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-    networks:
-      - backend
-    environment:
-      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - PGDATA=/var/lib/postgresql/data/db
-    volumes:
-      - pg_data:/var/lib/postgresql/data
-    healthcheck:
-      test: ["CMD", "pg_isready", "-U", "ckan"]
-
-  solr:
-    container_name: solr
-    image: ckan/ckan-solr:2.10-solr9-spatial
-    platform: linux/amd64
-    networks:
-      - backend
-    env_file:
-      - ./.ckan.env
-    environment:
-      - CKAN_CORE_NAME=${CKAN_CORE_NAME}
-      - CKAN_VERSION=${CKAN_VERSION}
-    volumes:
-      - solr_data:/var/solr
-      - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
-    healthcheck:
-      test: ["CMD-SHELL", "/opt/solr/bin/solr","status"]
-      interval: 5s
-      timeout: 5s
-      retries: 5
-      start_period: 5s
-
-  redis:
-    platform: linux/amd64
-    container_name: redis
-    image: redis:${REDIS_VERSION}
-    networks:
-      - backend
-    healthcheck:
-      test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-      start_period: 30s
-
-networks:
-  frontend:
-  backend:

compose/2.10/postgresql/Dockerfile

@@ -1,13 +0,0 @@
-FROM postgis/postgis:14-3.2-alpine
-
-# Allow connections; we don't map out any ports so only linked docker containers can connect
-RUN echo "host all  all    0.0.0.0/0  md5" >> /var/lib/postgresql/data/pg_hba.conf
-
-# Customize default user/pass/db
-ENV POSTGRES_DB ckan
-ENV POSTGRES_USER ckan
-ARG POSTGRES_PASSWORD
-ARG DS_RO_PASS
-
-# Include datastore setup scripts
-COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d

compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-set -e
-
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
-    CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
-    CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
-    GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;
-EOSQL

compose/2.10/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql

@@ -1,2 +0,0 @@
-ALTER VIEW geometry_columns OWNER TO ckan;
-ALTER TABLE spatial_ref_sys OWNER TO ckan;

compose/2.10/requirements.txt

@@ -1,3 +0,0 @@
-# Spatial
--e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial
--r https://raw.githubusercontent.com/ckan/ckanext-spatial/v2.1.1/requirements.txt

compose/2.10/solr8/ckan_init_solr.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# Initialize SOLR for CKAN by creating a ckan core
-# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION
-# Example:
-#   CKAN_CORE_NAME=ckan
-#   CKAN_VERSION=2.9.7
-
-set -e
-
-MAJOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 1`
-MINOR_VERSION=`echo $CKAN_VERSION | cut -d "." -f 2`
-
-CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/`echo $MAJOR_VERSION.$MINOR_VERSION`/ckan/config/solr/schema.xml
-
-echo "Check whether managed schema exists for CKAN $CKAN_VERSION"
-if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then
-  echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..."
-  exit 1
-fi
-
-echo "Check whether SOLR is initialized for CKAN"
-CORESDIR=/var/solr/data
-
-COREDIR="$CORESDIR/$CKAN_CORE_NAME"
-if [ -d "$COREDIR" ]; then
-    echo "SOLR already initialized, skipping initialization"
-else
-    echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION"
-    # init script for handling an empty /var/solr
-    /opt/solr-9.2.1/docker/scripts/init-var-solr
-    
-    # Precreate CKAN core
-    /opt/solr-9.2.1/docker/scripts/precreate-core $CKAN_CORE_NAME
-    
-    # Replace the managed schema with CKANs schema
-    echo "Adding CKAN managed schema"
-    curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s
-    
-    echo "SOLR initialized"
-fi

compose/2.11/.ckan-env

@@ -2,6 +2,8 @@
 # Information about how it works: https://github.com/okfn/ckanext-envvars
 # Note that variables here take presedence over build/up time variables in .env
 
+DEVELOPMENT=true
+
 # Set to true to disable CKAN from starting and serve a maintenance page
 MAINTENANCE_MODE=false
 
@@ -9,13 +11,13 @@ MAINTENANCE_MODE=false
 CKAN_SITE_ID=default
 CKAN_SITE_URL=http://localhost:5000
 CKAN_PORT=5000
-CKAN_MAX_UPLOAD_SIZE_MB=20
+CKAN_MAX_UPLOAD_SIZE_MB=200
 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
 # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
 CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
 CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
 # CKAN Plugins
-CKAN__PLUGINS=envvars activity image_view
+#CKAN__PLUGINS="envvars image_view text_view datatables_view datastore datapusher spatial_metadata spatial_query d4science d4science_theme"
 # CKAN requires storage path to be set in order for filestore to be enabled
 CKAN__STORAGE_PATH=/srv/app/data
 CKAN__WEBASSETS__PATH=/srv/app/data/webassets
@@ -39,4 +41,4 @@ CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0l
 
 # Solr configuration
 CKAN_VERSION=2.11
-CKAN_CORE_NAME=ckan
+CKAN_CORE_NAME=ckan

compose/2.11/Dockerfile

@@ -1,22 +1,22 @@
 ###################
 ### Extensions
 ###################
-FROM d4science/ckan:2.10.4-focal AS extbuild
+FROM d4science/ckan:2.11.0-focal AS extbuild
 
 # Switch to the root user
 USER root
 
 #install pip
-RUN apt-get update
-RUN apt-get install python3-dev python3-pip libxml2-dev libxslt1-dev libgeos-c1v5 -y
+#RUN apt-get update
+#RUN apt-get install python3-dev python3-pip libxml2-dev libxslt1-dev libgeos-c1v5 -y
 
 ############
 ### MAIN
 ############
-FROM d4science/ckan:2.10.4-focal
+FROM d4science/ckan:2.11.0-focal
 
 # Add the custom extensions to the plugins list
-ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher spatial_metadata spatial_query d4science d4science_theme
+ENV CKAN__PLUGINS="envvars tracking image_view text_view datatables_view datastore datapusher spatial_metadata spatial_query d4science d4science_theme"
 
 # Switch to the root user
 USER root

compose/2.11/docker-compose.yml

@@ -7,7 +7,8 @@ volumes:
 services:
   ckan:
     container_name: ckan
-    image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION}
+    build:
+      context: .
     networks:
       - frontend
       - backend
@@ -34,6 +35,8 @@ services:
       - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
     volumes:
       - ckan_data:/srv/app/data
+      - ./src:/srv/app/src_extensions
+      - ./docker-entrypoint.d:/srv/app/docker-entrypoint.d
 
   datapusher:
     container_name: datapusher
@@ -74,7 +77,7 @@ services:
 
   solr:
     container_name: solr
-    image: solr:8.11.1
+    image: ckan/ckan-solr:2.11-solr9-spatial
     networks:
       - backend
     env_file:

compose/2.11/requirements.txt

@@ -0,0 +1,3 @@
+# Spatial
+-e git+https://github.com/mjanez/ckanext-spatial@v2.1.1#egg=ckanext-spatial
+-r https://raw.githubusercontent.com/mjanez/ckanext-spatial/master/requirements.txt

compose/2.11/src/ckanext-d4science/setup.cfg

@@ -19,7 +19,6 @@ keywords = CKAN
 
 [options]
 packages = find:
-namespace_packages = ckanext
 install_requires =
 include_package_data = True
 

compose/2.11/src/ckanext-d4science_theme/ckanext/d4science_theme/helpers.py

@@ -25,7 +25,6 @@ import collections
 import ckan.plugins.toolkit as tk
 import ckan.logic as logic
 
-
 log = getLogger(__name__)
 
 systemtype_field = 'systemtypefield'
@@ -48,6 +47,7 @@ TRANSLATE_OF_ = 'translate_of_'
 
 ctg_namespace_ctrl = None
 
+
 # def get_tag_list()
 #     return logic.get_action('tag_list')
 
@@ -162,7 +162,8 @@ def get_d4s_namespace_controller():
         if the_namespaces is None or len(the_namespaces) == 0:
             log.info("D4S_Namespaces_Controller obj with none or empty namespaces, going to read them")
         else:
-            log.info("d4s_namespaces_controller found and the namespaces property is not empty: %s" % d4s_extras_controller)
+            log.info(
+                "d4s_namespaces_controller found and the namespaces property is not empty: %s" % d4s_extras_controller)
             return d4s_extras_controller
     else:
         log.info("ctg_namespace_ctrl with configurations is None, instancing it")
@@ -495,7 +496,7 @@ def get_list_of_groups(limit=10, sort='package_count'):
         ordered_groups = []
         ordered_groups = logic.get_action('group_list')({}, data)
 
-        log.debug("group_list: "+ ", ".join(ordered_groups))
+        log.debug("group_list: " + ", ".join(ordered_groups))
 
         for group in ordered_groups:
             # print "\n\ngroup %s" %group
@@ -718,6 +719,14 @@ def get_location_to_bboxes():
     except Exception as error:
         log.error("Error on reading file: %s" % bboxes_file + "error: %s" % error)
 
+
 def get_content_moderator_system_placeholder():
     return systemtype_cms_fields_placeholders
 
+
+def get_site_statistics() -> dict[str, int]:
+    return {
+        'dataset_count': logic.get_action('package_search')({}, {"rows": 1})['count'],
+        'group_count': len(logic.get_action('group_list')({}, {})),
+        'organization_count': len(logic.get_action('organization_list')({}, {}))
+    }

compose/2.11/src/ckanext-d4science_theme/ckanext/d4science_theme/plugin.py

@@ -2,18 +2,14 @@
 from logging import getLogger
 
 import ckan.plugins as plugins
+
 from ckanext.d4science_theme import helpers
 import ckan.plugins.toolkit as toolkit
-import ckan.lib.dictization.model_save as model_save
 import ckan.model as model
-import ckan.lib.helpers as h
-import sqlalchemy as sa
-from ckanext.d4science_theme.controllers.organization import OrganizationVREController
 from ckanext.d4science_theme.controllers.home import d4SHomeController
 from ckanext.d4science_theme.controllers.systemtype import d4STypeController
 from ckanext.d4science_theme.controllers.organization import OrganizationVREController
 #from ckan.controllers.home import HomeController
-from ckan.config.middleware.common_middleware import TrackingMiddleware
 #from ckan.plugins import IRoutes
 from flask import Blueprint, render_template
 
@@ -194,32 +190,6 @@ def get_package_for_value(list_package, value):
     
     return lst
 
-#OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE
-def _init_TrackingMiddleware(self, app, config):
-    self.app = app
-    log.debug('TrackingMiddleware d4Science instance')
-    sqlalchemy_url = config.get('sqlalchemy.url')
-    log.debug('sqlalchemy_url read: '+str(sqlalchemy_url))
-    
-    sqlalchemy_pool = config.get('sqlalchemy.pool_size')
-    if sqlalchemy_pool is None:
-        sqlalchemy_pool = 5
-    
-    log.debug('sqlalchemy_pool read: '+str(sqlalchemy_pool))
-    sqlalchemy_overflow = config.get('sqlalchemy.max_overflow')
-    
-    if sqlalchemy_overflow is None:
-        sqlalchemy_overflow = 10;
-    
-    log.debug('sqlalchemy_overflow read: '+str(sqlalchemy_overflow))
-    
-    try:
-        self.engine = sa.create_engine(sqlalchemy_url, pool_size=int(sqlalchemy_pool), max_overflow=int(sqlalchemy_overflow))
-    except TypeError as e:
-        log.error('pool size does not work: ' +str(e.args))
-        self.engine = sa.create_engine(sqlalchemy_url)
-            
-
 
 class D4Science_ThemePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
     plugins.implements(plugins.IConfigurer)
@@ -319,6 +289,7 @@ class D4Science_ThemePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm)
             'd4science_get_ckan_translate_for' : helpers.get_ckan_translate_for,
             'd4science_get_location_to_bboxes' : helpers.get_location_to_bboxes,
             'd4science_get_content_moderator_system_placeholder': helpers.get_content_moderator_system_placeholder,
+            'd4science_get_site_statistics': helpers.get_site_statistics,
         }
     
     #Overriding package_extras_save method
@@ -328,9 +299,6 @@ class D4Science_ThemePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm)
     #Overriding index home controller - rimosso in ckan 2.10
     #d4sHC = d4SHomeController()
     # HomeController.index = d4sHC.index
-    
-    #OVERRIDING BASE SQL ALCHEMY ENGINE INSTANCE
-    TrackingMiddleware.__init__ = _init_TrackingMiddleware
 
     global d4s_ctg_namespaces_controller