CatalogueRevamping
/
docker-ckan
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

deleted unused files

This commit is contained in:
Elia Bellavista 2024-07-16 18:40:41 +02:00
parent a9989167f9
commit d7ee7d66f3
66 changed files with 0 additions and 4506 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
compose/2.8/.ckan-env
View File

@ -1,36 +0,0 @@
# Runtime configuration of CKAN enabled through ckanext-envvars
# Information about how it works: https://github.com/okfn/ckanext-envvars
# Note that variables here take presedence over build/up time variables in .env
# Set to true to disable CKAN from starting and serve a maintenance page
MAINTENANCE_MODE=false
# General Settings
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000
CKAN_MAX_UPLOAD_SIZE_MB=10
CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
# CKAN Plugins
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
# CKAN requires storage path to be set in order for filestore to be enabled
CKAN__STORAGE_PATH=/srv/app/data
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
CKAN_SYSADMIN_NAME=sysadmin
CKAN_SYSADMIN_PASSWORD=password
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
# Email settings
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
# Datapusher configuration
CKAN__DATAPUSHER__URL=http://datapusher:8000
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/

32
compose/2.8/.env
View File

@ -1,32 +0,0 @@
# Variables in this file will be used as build arguments when running
# docker-compose build and docker-compose up
# Verify correct substitution with "docker-compose config"
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
# docker-compose down -v
# docker-compose build
# docker-compose up -d
# Database
POSTGRES_PASSWORD=ckan
POSTGRES_PORT=5432
DATASTORE_READONLY_PASSWORD=datastore
# CKAN
CKAN_VERSION=2.8.12
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000
CKAN_MAX_UPLOAD_SIZE_MB=10
# Datapusher
DATAPUSHER_VERSION=0.0.17
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
DATAPUSHER_CHUNK_SIZE=16384
DATAPUSHER_CHUNK_INSERT_ROWS=250
DATAPUSHER_DOWNLOAD_TIMEOUT=30
DATAPUSHER_SSL_VERIFY=False
DATAPUSHER_REWRITE_RESOURCES=True
DATAPUSHER_REWRITE_URL=http://ckan:5000
# Redis
REDIS_VERSION=6.0.7

93
compose/2.8/docker-compose.yml
View File

@ -1,93 +0,0 @@
# docker-compose build && docker-compose up -d
version: "3"
volumes:
ckan_data:
pg_data:
solr_data:
services:
ckan:
container_name: ckan
image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION}
networks:
- frontend
- backend
depends_on:
- db
- solr
ports:
- "0.0.0.0:${CKAN_PORT}:5000"
env_file:
- ./.ckan-env
environment:
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
volumes:
- ckan_data:/srv/app/data
datapusher:
container_name: datapusher
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
networks:
- frontend
- backend
ports:
- "8000:8000"
environment:
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
- DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
db:
container_name: db
build:
context: .
dockerfile: postgresql/Dockerfile
args:
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
networks:
- backend
environment:
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- PGDATA=/var/lib/postgresql/data/db
volumes:
- pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "ckan"]
solr:
container_name: solr
build:
context: .
dockerfile: solr/Dockerfile
args:
- CKAN_VERSION=${CKAN_VERSION}
networks:
- backend
volumes:
- solr_data:/opt/solr/server/solr/ckan/data
redis:
container_name: redis
image: redis:${REDIS_VERSION}
networks:
- backend
networks:
frontend:
backend:

13
compose/2.8/postgresql/Dockerfile
View File

@ -1,13 +0,0 @@
FROM postgis/postgis:11-3.3-alpine
# Allow connections; we don't map out any ports so only linked docker containers can connect
RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf
# Customize default user/pass/db
ENV POSTGRES_DB ckan
ENV POSTGRES_USER ckan
ARG POSTGRES_PASSWORD
ARG DS_RO_PASS
# Include datastore setup scripts
COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d

8
compose/2.8/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh
View File

@ -1,8 +0,0 @@
#!/bin/bash
set -e
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;
EOSQL

2
compose/2.8/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql
View File

@ -1,2 +0,0 @@
ALTER VIEW geometry_columns OWNER TO ckan;
ALTER TABLE spatial_ref_sys OWNER TO ckan;

35
compose/2.8/solr/Dockerfile
View File

@ -1,35 +0,0 @@
FROM solr:6.6.6
# Enviroment
ENV SOLR_CORE ckan
ENV SOLR_VERSION 6.6.6
# Build Arguments
ARG CKAN_VERSION
# Create Directories
RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf
RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data
# Adding Files
COPY ./solr/solrconfig-$CKAN_VERSION.xml /opt/solr/server/solr/$SOLR_CORE/conf/
ADD https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.xml \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
/opt/solr/server/solr/$SOLR_CORE/conf/
# Create Core.properties
RUN mv /opt/solr/server/solr/$SOLR_CORE/conf/solrconfig-$CKAN_VERSION.xml /opt/solr/server/solr/$SOLR_CORE/conf/solrconfig.xml && \
echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties
# Giving ownership to Solr
USER root
RUN chown -R "$SOLR_USER:$SOLR_USER" /opt/solr/server/solr/$SOLR_CORE
# User
USER $SOLR_USER:$SOLR_USER

1
compose/2.8/solr/solrconfig-2.7.12.xml
View File

@ -1 +0,0 @@
solrconfig-2.8.10.xml

343
compose/2.8/solr/solrconfig-2.8.12.xml
View File

@ -1,343 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--solrconfig.xml documentation [https://wiki.apache.org/solr/SolrConfigXml]-->
<config>
<luceneMatchVersion>6.0.0</luceneMatchVersion>
<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}" />
<codecFactory class="solr.SchemaCodecFactory" />
<indexConfig>
<lockType>${solr.lock.type:native}</lockType>
</indexConfig>
<jmx />
<updateHandler class="solr.DirectUpdateHandler2">
<updateLog>
<str name="dir">${solr.ulog.dir:}</str>
<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
</updateLog>
<autoCommit>
<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
<openSearcher>false</openSearcher>
</autoCommit>
<autoSoftCommit>
<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
</autoSoftCommit>
</updateHandler>
<query>
<maxBooleanClauses>1024</maxBooleanClauses>
<filterCache class="solr.FastLRUCache" size="512" initialSize="512" autowarmCount="0" />
<queryResultCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
<documentCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
<cache name="perSegFilter" class="solr.search.LRUCache" size="10" initialSize="0" autowarmCount="10" regenerator="solr.NoOpRegenerator" />
<enableLazyFieldLoading>true</enableLazyFieldLoading>
<queryResultWindowSize>20</queryResultWindowSize>
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
<listener event="newSearcher" class="solr.QuerySenderListener">
<arr name="queries" />
</listener>
<listener event="firstSearcher" class="solr.QuerySenderListener">
<arr name="queries" />
</listener>
<useColdSearcher>false</useColdSearcher>
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>
<requestDispatcher handleSelect="false">
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048" addHttpRequestToContext="false" />
<httpCaching never304="true" />
</requestDispatcher>
<requestHandler name="/select" class="solr.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
<int name="rows">10</int>
</lst>
</requestHandler>
<requestHandler name="/query" class="solr.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
<str name="wt">json</str>
<str name="indent">true</str>
</lst>
</requestHandler>
<requestHandler name="/browse" class="solr.SearchHandler" useParams="query,facets,velocity,browse">
<lst name="defaults">
<str name="echoParams">explicit</str>
</lst>
</requestHandler>
<initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
<lst name="defaults">
<str name="df">_text_</str>
</lst>
</initParams>
<initParams path="/update/**">
<lst name="defaults">
<str name="update.chain">add-unknown-fields-to-the-schema</str>
</lst>
</initParams>
<requestHandler name="/update/extract" startup="lazy" class="solr.extraction.ExtractingRequestHandler">
<lst name="defaults">
<str name="lowernames">true</str>
<str name="fmap.meta">ignored_</str>
<str name="fmap.content">_text_</str>
</lst>
</requestHandler>
<requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
<requestHandler name="/analysis/document" class="solr.DocumentAnalysisRequestHandler" startup="lazy" />
<requestHandler name="/debug/dump" class="solr.DumpRequestHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
<str name="echoHandler">true</str>
</lst>
</requestHandler>
<searchComponent name="spellcheck" class="solr.SpellCheckComponent">
<str name="queryAnalyzerFieldType">text_general</str>
<lst name="spellchecker">
<str name="name">default</str>
<str name="field">_text_</str>
<str name="classname">solr.DirectSolrSpellChecker</str>
<str name="distanceMeasure">internal</str>
<float name="accuracy">0.5</float>
<int name="maxEdits">2</int>
<int name="minPrefix">1</int>
<int name="maxInspections">5</int>
<int name="minQueryLength">4</int>
<float name="maxQueryFrequency">0.01</float>
</lst>
</searchComponent>
<requestHandler name="/spell" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<str name="spellcheck.dictionary">default</str>
<str name="spellcheck">on</str>
<str name="spellcheck.extendedResults">true</str>
<str name="spellcheck.count">10</str>
<str name="spellcheck.alternativeTermCount">5</str>
<str name="spellcheck.maxResultsForSuggest">5</str>
<str name="spellcheck.collate">true</str>
<str name="spellcheck.collateExtendedResults">true</str>
<str name="spellcheck.maxCollationTries">10</str>
<str name="spellcheck.maxCollations">5</str>
</lst>
<arr name="last-components">
<str>spellcheck</str>
</arr>
</requestHandler>
<searchComponent name="tvComponent" class="solr.TermVectorComponent" />
<requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<bool name="tv">true</bool>
</lst>
<arr name="last-components">
<str>tvComponent</str>
</arr>
</requestHandler>
<searchComponent name="terms" class="solr.TermsComponent" />
<requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<bool name="terms">true</bool>
<bool name="distrib">false</bool>
</lst>
<arr name="components">
<str>terms</str>
</arr>
</requestHandler>
<searchComponent name="elevator" class="solr.QueryElevationComponent">
<str name="queryFieldType">string</str>
<str name="config-file">elevate.xml</str>
</searchComponent>
<requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<str name="echoParams">explicit</str>
</lst>
<arr name="last-components">
<str>elevator</str>
</arr>
</requestHandler>
<searchComponent class="solr.HighlightComponent" name="highlight">
<highlighting>
<fragmenter name="gap" default="true" class="solr.highlight.GapFragmenter">
<lst name="defaults">
<int name="hl.fragsize">100</int>
</lst>
</fragmenter>
<fragmenter name="regex" class="solr.highlight.RegexFragmenter">
<lst name="defaults">
<int name="hl.fragsize">70</int>
<float name="hl.regex.slop">0.5</float>
<str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str>
</lst>
</fragmenter>
<formatter name="html" default="true" class="solr.highlight.HtmlFormatter">
<lst name="defaults">
<str name="hl.simple.pre"><![CDATA[<em>]]></str>
<str name="hl.simple.post"><![CDATA[</em>]]></str>
</lst>
</formatter>
<encoder name="html" class="solr.highlight.HtmlEncoder" />
<fragListBuilder name="simple" class="solr.highlight.SimpleFragListBuilder" />
<fragListBuilder name="single" class="solr.highlight.SingleFragListBuilder" />
<fragListBuilder name="weighted" default="true" class="solr.highlight.WeightedFragListBuilder" />
<fragmentsBuilder name="default" default="true" class="solr.highlight.ScoreOrderFragmentsBuilder" />
<fragmentsBuilder name="colored" class="solr.highlight.ScoreOrderFragmentsBuilder">
<lst name="defaults">
<str name="hl.tag.pre"><![CDATA[<b style="background:yellow">,<b style="background:lawgreen">,
<b style="background:aquamarine">,<b style="background:magenta">,
<b style="background:palegreen">,<b style="background:coral">,
<b style="background:wheat">,<b style="background:khaki">,
<b style="background:lime">,<b style="background:deepskyblue">]]></str>
<str name="hl.tag.post"><![CDATA[</b>]]></str>
</lst>
</fragmentsBuilder>
<boundaryScanner name="default" default="true" class="solr.highlight.SimpleBoundaryScanner">
<lst name="defaults">
<str name="hl.bs.maxScan">10</str>
<str name="hl.bs.chars">.,!?</str>
</lst>
</boundaryScanner>
<boundaryScanner name="breakIterator" class="solr.highlight.BreakIteratorBoundaryScanner">
<lst name="defaults">
<str name="hl.bs.type">WORD</str>
<str name="hl.bs.language">en</str>
<str name="hl.bs.country">US</str>
</lst>
</boundaryScanner>
</highlighting>
</searchComponent>
<schemaFactory class="ClassicIndexSchemaFactory" />
<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
<processor class="solr.UUIDUpdateProcessorFactory" />
<processor class="solr.LogUpdateProcessorFactory" />
<processor class="solr.DistributedUpdateProcessorFactory" />
<processor class="solr.RemoveBlankFieldUpdateProcessorFactory" />
<processor class="solr.FieldNameMutatingUpdateProcessorFactory">
<str name="pattern">[^\w-\.]</str>
<str name="replacement">_</str>
</processor>
<processor class="solr.ParseBooleanFieldUpdateProcessorFactory" />
<processor class="solr.ParseLongFieldUpdateProcessorFactory" />
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory" />
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
<str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
<str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
<str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
<str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
<str>yyyy-MM-dd'T'HH:mm:ssZ</str>
<str>yyyy-MM-dd'T'HH:mm:ss</str>
<str>yyyy-MM-dd'T'HH:mmZ</str>
<str>yyyy-MM-dd'T'HH:mm</str>
<str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
<str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
<str>yyyy-MM-dd HH:mm:ss.SSS</str>
<str>yyyy-MM-dd HH:mm:ss,SSS</str>
<str>yyyy-MM-dd HH:mm:ssZ</str>
<str>yyyy-MM-dd HH:mm:ss</str>
<str>yyyy-MM-dd HH:mmZ</str>
<str>yyyy-MM-dd HH:mm</str>
<str>yyyy-MM-dd</str>
</arr>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
<queryResponseWriter name="json" class="solr.JSONResponseWriter">
<str name="content-type">text/plain; charset=UTF-8</str>
</queryResponseWriter>
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter" startup="lazy">
<str name="template.base.dir">${velocity.template.base.dir:}</str>
<str name="solr.resource.loader.enabled">${velocity.solr.resource.loader.enabled:true}</str>
<str name="params.resource.loader.enabled">${velocity.params.resource.loader.enabled:false}</str>
</queryResponseWriter>
<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
<int name="xsltCacheLifetimeSeconds">5</int>
</queryResponseWriter>
</config>

345
compose/2.8/solr/solrconfig-2.9.7.xml
View File

@ -1,345 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--solrconfig.xml documentation [https://wiki.apache.org/solr/SolrConfigXml]-->
<config>
<luceneMatchVersion>6.0.0</luceneMatchVersion>
<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}" />
<codecFactory class="solr.SchemaCodecFactory" />
<indexConfig>
<lockType>${solr.lock.type:native}</lockType>
</indexConfig>
<jmx />
<updateHandler class="solr.DirectUpdateHandler2">
<updateLog>
<str name="dir">${solr.ulog.dir:}</str>
<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
</updateLog>
<autoCommit>
<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
<openSearcher>false</openSearcher>
</autoCommit>
<autoSoftCommit>
<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
</autoSoftCommit>
</updateHandler>
<query>
<maxBooleanClauses>1024</maxBooleanClauses>
<filterCache class="solr.FastLRUCache" size="512" initialSize="512" autowarmCount="0" />
<queryResultCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
<documentCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
<cache name="perSegFilter" class="solr.search.LRUCache" size="10" initialSize="0" autowarmCount="10" regenerator="solr.NoOpRegenerator" />
<enableLazyFieldLoading>true</enableLazyFieldLoading>
<queryResultWindowSize>20</queryResultWindowSize>
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
<listener event="newSearcher" class="solr.QuerySenderListener">
<arr name="queries" />
</listener>
<listener event="firstSearcher" class="solr.QuerySenderListener">
<arr name="queries" />
</listener>
<useColdSearcher>false</useColdSearcher>
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>
<requestDispatcher handleSelect="false">
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048" addHttpRequestToContext="false" />
<httpCaching never304="true" />
</requestDispatcher>
<requestHandler name="/select" class="solr.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
<int name="rows">10</int>
</lst>
</requestHandler>
<requestHandler name="/query" class="solr.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
<str name="wt">json</str>
<str name="indent">true</str>
</lst>
</requestHandler>
<requestHandler name="/browse" class="solr.SearchHandler" useParams="query,facets,velocity,browse">
<lst name="defaults">
<str name="echoParams">explicit</str>
</lst>
</requestHandler>
<initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
<lst name="defaults">
<str name="df">_text_</str>
</lst>
</initParams>
<initParams path="/update/**">
<lst name="defaults">
<str name="update.chain">add-unknown-fields-to-the-schema</str>
</lst>
</initParams>
<requestHandler name="/update/extract" startup="lazy" class="solr.extraction.ExtractingRequestHandler">
<lst name="defaults">
<str name="lowernames">true</str>
<str name="fmap.meta">ignored_</str>
<str name="fmap.content">_text_</str>
</lst>
</requestHandler>
<requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
<requestHandler name="/analysis/document" class="solr.DocumentAnalysisRequestHandler" startup="lazy" />
<requestHandler name="/debug/dump" class="solr.DumpRequestHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
<str name="echoHandler">true</str>
</lst>
</requestHandler>
<searchComponent name="spellcheck" class="solr.SpellCheckComponent">
<str name="queryAnalyzerFieldType">text_general</str>
<lst name="spellchecker">
<str name="name">default</str>
<str name="field">_text_</str>
<str name="classname">solr.DirectSolrSpellChecker</str>
<str name="distanceMeasure">internal</str>
<float name="accuracy">0.5</float>
<int name="maxEdits">2</int>
<int name="minPrefix">1</int>
<int name="maxInspections">5</int>
<int name="minQueryLength">4</int>
<float name="maxQueryFrequency">0.01</float>
</lst>
</searchComponent>
<requestHandler name="/spell" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<str name="spellcheck.dictionary">default</str>
<str name="spellcheck">on</str>
<str name="spellcheck.extendedResults">true</str>
<str name="spellcheck.count">10</str>
<str name="spellcheck.alternativeTermCount">5</str>
<str name="spellcheck.maxResultsForSuggest">5</str>
<str name="spellcheck.collate">true</str>
<str name="spellcheck.collateExtendedResults">true</str>
<str name="spellcheck.maxCollationTries">10</str>
<str name="spellcheck.maxCollations">5</str>
</lst>
<arr name="last-components">
<str>spellcheck</str>
</arr>
</requestHandler>
<searchComponent name="tvComponent" class="solr.TermVectorComponent" />
<requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<bool name="tv">true</bool>
</lst>
<arr name="last-components">
<str>tvComponent</str>
</arr>
</requestHandler>
<searchComponent name="terms" class="solr.TermsComponent" />
<requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<bool name="terms">true</bool>
<bool name="distrib">false</bool>
</lst>
<arr name="components">
<str>terms</str>
</arr>
</requestHandler>
<searchComponent name="elevator" class="solr.QueryElevationComponent">
<str name="queryFieldType">string</str>
<str name="config-file">elevate.xml</str>
</searchComponent>
<requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<str name="echoParams">explicit</str>
</lst>
<arr name="last-components">
<str>elevator</str>
</arr>
</requestHandler>
<searchComponent class="solr.HighlightComponent" name="highlight">
<highlighting>
<fragmenter name="gap" default="true" class="solr.highlight.GapFragmenter">
<lst name="defaults">
<int name="hl.fragsize">100</int>
</lst>
</fragmenter>
<fragmenter name="regex" class="solr.highlight.RegexFragmenter">
<lst name="defaults">
<int name="hl.fragsize">70</int>
<float name="hl.regex.slop">0.5</float>
<str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str>
</lst>
</fragmenter>
<formatter name="html" default="true" class="solr.highlight.HtmlFormatter">
<lst name="defaults">
<str name="hl.simple.pre"><![CDATA[<em>]]></str>
<str name="hl.simple.post"><![CDATA[</em>]]></str>
</lst>
</formatter>
<encoder name="html" class="solr.highlight.HtmlEncoder" />
<fragListBuilder name="simple" class="solr.highlight.SimpleFragListBuilder" />
<fragListBuilder name="single" class="solr.highlight.SingleFragListBuilder" />
<fragListBuilder name="weighted" default="true" class="solr.highlight.WeightedFragListBuilder" />
<fragmentsBuilder name="default" default="true" class="solr.highlight.ScoreOrderFragmentsBuilder" />
<fragmentsBuilder name="colored" class="solr.highlight.ScoreOrderFragmentsBuilder">
<lst name="defaults">
<str name="hl.tag.pre"><![CDATA[<b style="background:yellow">,<b style="background:lawgreen">,
<b style="background:aquamarine">,<b style="background:magenta">,
<b style="background:palegreen">,<b style="background:coral">,
<b style="background:wheat">,<b style="background:khaki">,
<b style="background:lime">,<b style="background:deepskyblue">]]></str>
<str name="hl.tag.post"><![CDATA[</b>]]></str>
</lst>
</fragmentsBuilder>
<boundaryScanner name="default" default="true" class="solr.highlight.SimpleBoundaryScanner">
<lst name="defaults">
<str name="hl.bs.maxScan">10</str>
<str name="hl.bs.chars">.,!?</str>
</lst>
</boundaryScanner>
<boundaryScanner name="breakIterator" class="solr.highlight.BreakIteratorBoundaryScanner">
<lst name="defaults">
<str name="hl.bs.type">WORD</str>
<str name="hl.bs.language">en</str>
<str name="hl.bs.country">US</str>
</lst>
</boundaryScanner>
</highlighting>
</searchComponent>
<schemaFactory class="ManagedIndexSchemaFactory">
<bool name="mutable">true</bool>
</schemaFactory>
<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
<processor class="solr.UUIDUpdateProcessorFactory" />
<processor class="solr.LogUpdateProcessorFactory" />
<processor class="solr.DistributedUpdateProcessorFactory" />
<processor class="solr.RemoveBlankFieldUpdateProcessorFactory" />
<processor class="solr.FieldNameMutatingUpdateProcessorFactory">
<str name="pattern">[^\w-\.]</str>
<str name="replacement">_</str>
</processor>
<processor class="solr.ParseBooleanFieldUpdateProcessorFactory" />
<processor class="solr.ParseLongFieldUpdateProcessorFactory" />
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory" />
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
<str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
<str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
<str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
<str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
<str>yyyy-MM-dd'T'HH:mm:ssZ</str>
<str>yyyy-MM-dd'T'HH:mm:ss</str>
<str>yyyy-MM-dd'T'HH:mmZ</str>
<str>yyyy-MM-dd'T'HH:mm</str>
<str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
<str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
<str>yyyy-MM-dd HH:mm:ss.SSS</str>
<str>yyyy-MM-dd HH:mm:ss,SSS</str>
<str>yyyy-MM-dd HH:mm:ssZ</str>
<str>yyyy-MM-dd HH:mm:ss</str>
<str>yyyy-MM-dd HH:mmZ</str>
<str>yyyy-MM-dd HH:mm</str>
<str>yyyy-MM-dd</str>
</arr>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
<queryResponseWriter name="json" class="solr.JSONResponseWriter">
<str name="content-type">text/plain; charset=UTF-8</str>
</queryResponseWriter>
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter" startup="lazy">
<str name="template.base.dir">${velocity.template.base.dir:}</str>
<str name="solr.resource.loader.enabled">${velocity.solr.resource.loader.enabled:true}</str>
<str name="params.resource.loader.enabled">${velocity.params.resource.loader.enabled:false}</str>
</queryResponseWriter>
<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
<int name="xsltCacheLifetimeSeconds">5</int>
</queryResponseWriter>
</config>

203
compose/2.8/solr/solrconfig-2.9.8.xml
View File

@ -1,203 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
NB Please copy changes to this file into the multilingual schema:
ckanext/multilingual/solr/schema.xml
-->
<!-- We update the version when there is a backward-incompatible change to this
schema. We used to use the `version` attribute for this but this is an internal
attribute that should not be used so starting from CKAN 2.10 we use the `name`
attribute with the form `ckan-X.Y` -->
<schema name="ckan-2.10" version="1.6">
<types>
<fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
<fieldtype name="binary" class="solr.BinaryField"/>
<fieldType name="int" class="solr.IntPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="float" class="solr.FloatPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="long" class="solr.LongPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="double" class="solr.DoublePointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pint" class="solr.IntPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pfloat" class="solr.FloatPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="plong" class="solr.LongPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pdouble" class="solr.DoublePointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="date" class="solr.DatePointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pdate" class="solr.DatePointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pdates" class="solr.DatePointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
<fieldType name="pints" class="solr.IntPointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="pfloats" class="solr.FloatPointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="plongs" class="solr.LongPointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="pdoubles" class="solr.DoublePointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="text" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
<filter class="solr.ASCIIFoldingFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
<filter class="solr.ASCIIFoldingFilterFactory"/>
</analyzer>
</fieldType>
<!-- A general unstemmed text field - good if one does not know the language of the field -->
<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
<filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<fieldType name="text_ngram" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.NGramTokenizerFactory" minGramSize="2" maxGramSize="10"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
</types>
<fields>
<field name="index_id" type="string" indexed="true" stored="true" required="true" />
<field name="id" type="string" indexed="true" stored="true" required="true" />
<field name="site_id" type="string" indexed="true" stored="true" required="true" />
<field name="title" type="text" indexed="true" stored="true" />
<field name="title_ngram" type="text_ngram" indexed="true" stored="true" />
<field name="entity_type" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="dataset_type" type="string" indexed="true" stored="true" />
<field name="state" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="name" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="name_ngram" type="text_ngram" indexed="true" stored="true" />
<field name="revision_id" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="version" type="string" indexed="true" stored="true" />
<field name="url" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="ckan_url" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="download_url" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="notes" type="text" indexed="true" stored="true"/>
<field name="author" type="text_general" indexed="true" stored="true" />
<field name="author_email" type="text_general" indexed="true" stored="true" />
<field name="maintainer" type="text_general" indexed="true" stored="true" />
<field name="maintainer_email" type="text_general" indexed="true" stored="true" />
<field name="license" type="string" indexed="true" stored="true" />
<field name="license_id" type="string" indexed="true" stored="true" />
<field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="organization" type="string" indexed="true" stored="true" multiValued="false"/>
<field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
<field name="permission_labels" type="string" indexed="true" stored="false" multiValued="true"/>
<field name="res_name" type="text_general" indexed="true" stored="true" multiValued="true" />
<field name="res_description" type="text_general" indexed="true" stored="true" multiValued="true"/>
<field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="res_type" type="string" indexed="true" stored="true" multiValued="true"/>
<!-- catchall field, containing all other searchable text fields (implemented
via copyField further on in this schema -->
<field name="text" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="urls" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="depends_on" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="dependency_of" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="derives_from" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="has_derivation" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="links_to" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="views_total" type="int" indexed="true" stored="false"/>
<field name="views_recent" type="int" indexed="true" stored="false"/>
<field name="resources_accessed_total" type="int" indexed="true" stored="false"/>
<field name="resources_accessed_recent" type="int" indexed="true" stored="false"/>
<field name="metadata_created" type="date" indexed="true" stored="true" multiValued="false"/>
<field name="metadata_modified" type="date" indexed="true" stored="true" multiValued="false"/>
<field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
<!-- Copy the title field into titleString, and treat as a string
(rather than text type). This allows us to sort on the titleString -->
<field name="title_string" type="string" indexed="true" stored="false" />
<field name="data_dict" type="string" indexed="false" stored="true" />
<field name="validated_data_dict" type="string" indexed="false" stored="true" />
<field name="_version_" type="string" indexed="true" stored="true"/>
<dynamicField name="*_date" type="date" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="res_extras_*" type="text" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="vocab_*" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*" type="string" indexed="true" stored="false"/>
</fields>
<uniqueKey>index_id</uniqueKey>
<copyField source="url" dest="urls"/>
<copyField source="title" dest="title_ngram"/>
<copyField source="name" dest="name_ngram"/>
<copyField source="ckan_url" dest="urls"/>
<copyField source="download_url" dest="urls"/>
<copyField source="res_url" dest="urls"/>
<copyField source="extras_*" dest="text"/>
<copyField source="res_extras_*" dest="text"/>
<copyField source="vocab_*" dest="text"/>
<copyField source="urls" dest="text"/>
<copyField source="name" dest="text"/>
<copyField source="title" dest="text"/>
<copyField source="text" dest="text"/>
<copyField source="license" dest="text"/>
<copyField source="notes" dest="text"/>
<copyField source="tags" dest="text"/>
<copyField source="groups" dest="text"/>
<copyField source="organization" dest="text"/>
<copyField source="res_name" dest="text"/>
<copyField source="res_description" dest="text"/>
<copyField source="maintainer" dest="text"/>
<copyField source="author" dest="text"/>
</schema>

40
compose/2.9/.ckan-env
View File

@ -1,40 +0,0 @@
# Runtime configuration of CKAN enabled through ckanext-envvars
# Information about how it works: https://github.com/okfn/ckanext-envvars
# Note that variables here take presedence over build/up time variables in .env
# Set to true to disable CKAN from starting and serve a maintenance page
MAINTENANCE_MODE=false
# General Settings
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000
CKAN_MAX_UPLOAD_SIZE_MB=10
CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
# CKAN Plugins
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
# CKAN requires storage path to be set in order for filestore to be enabled
CKAN__STORAGE_PATH=/srv/app/data
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
CKAN_SYSADMIN_NAME=sysadmin
CKAN_SYSADMIN_PASSWORD=password
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
# Email settings
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
# Datapusher configuration
CKAN__DATAPUSHER__URL=http://datapusher:8000
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
# Solr configuration
CKAN_VERSION=2.9.11
CKAN_CORE_NAME=ckan

35
compose/2.9/.env
View File

@ -1,35 +0,0 @@
# Variables in this file will be used as build arguments when running
# docker-compose build and docker-compose up
# Verify correct substitution with "docker-compose config"
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
# docker-compose down -v
# docker-compose build
# docker-compose up -d
# Database
POSTGRES_PASSWORD=ckan
POSTGRES_PORT=5432
DATASTORE_READONLY_PASSWORD=datastore
# CKAN
CKAN_VERSION=2.9.11
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000
CKAN_MAX_UPLOAD_SIZE_MB=10
# Datapusher
DATAPUSHER_VERSION=0.0.17
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
DATAPUSHER_CHUNK_SIZE=16384
DATAPUSHER_CHUNK_INSERT_ROWS=250
DATAPUSHER_DOWNLOAD_TIMEOUT=30
DATAPUSHER_SSL_VERIFY=False
DATAPUSHER_REWRITE_RESOURCES=True
DATAPUSHER_REWRITE_URL=http://ckan:5000
# SOLR
CKAN_CORE_NAME=ckan
# Redis
REDIS_VERSION=6.0.7

95
compose/2.9/docker-compose.yml
View File

@ -1,95 +0,0 @@
# docker-compose build && docker-compose up -d
version: "3"
volumes:
ckan_data:
pg_data:
solr_data:
services:
ckan:
container_name: ckan
image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION}
networks:
- frontend
- backend
depends_on:
- db
- solr
ports:
- "0.0.0.0:${CKAN_PORT}:5000"
env_file:
- ./.ckan-env
environment:
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
volumes:
- ckan_data:/srv/app/data
datapusher:
container_name: datapusher
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
networks:
- frontend
- backend
ports:
- "8000:8000"
environment:
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
- DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
db:
container_name: db
build:
context: .
dockerfile: postgresql/Dockerfile
args:
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
networks:
- backend
environment:
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- PGDATA=/var/lib/postgresql/data/db
volumes:
- pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "ckan"]
solr:
container_name: solr
image: solr:8.11.1
networks:
- backend
env_file:
- ./.ckan-env
environment:
- CKAN_CORE_NAME=${CKAN_CORE_NAME}
- CKAN_VERSION=${CKAN_VERSION}
volumes:
- solr_data:/var/solr
- ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
redis:
container_name: redis
image: redis:${REDIS_VERSION}
networks:
- backend
networks:
frontend:
backend:

13
compose/2.9/postgresql/Dockerfile
View File

@ -1,13 +0,0 @@
FROM postgis/postgis:14-3.2-alpine
# Allow connections; we don't map out any ports so only linked docker containers can connect
RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf
# Customize default user/pass/db
ENV POSTGRES_DB ckan
ENV POSTGRES_USER ckan
ARG POSTGRES_PASSWORD
ARG DS_RO_PASS
# Include datastore setup scripts
COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d

8
compose/2.9/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh
View File

@ -1,8 +0,0 @@
#!/bin/bash
set -e
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;
EOSQL

2
compose/2.9/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql
View File

@ -1,2 +0,0 @@
ALTER VIEW geometry_columns OWNER TO ckan;
ALTER TABLE spatial_ref_sys OWNER TO ckan;

38
compose/2.9/solr8/ckan_init_solr.sh
View File

@ -1,38 +0,0 @@
#!/bin/bash
#
# Initialize SOLR for CKAN by creating a ckan core
# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION
# Example:
# CKAN_CORE_NAME=ckan
# CKAN_VERSION=2.9.11
set -e
CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.solr8.xml
echo "Check whether managed schema exists for CKAN $CKAN_VERSION"
if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then
echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..."
exit 1
fi
echo "Check whether SOLR is initialized for CKAN"
CORESDIR=/var/solr/data
COREDIR="$CORESDIR/$CKAN_CORE_NAME"
if [ -d "$COREDIR" ]; then
echo "SOLR already initialized, skipping initialization"
else
echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION"
# init script for handling an empty /var/solr
/opt/docker-solr/scripts/init-var-solr
# Precreate CKAN core
/opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME
# Replace the managed schema with CKANs schema
echo "Adding CKAN managed schema"
curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s
echo "SOLR initialized"
fi

37
examples/harvest/.ckan-env
View File

@ -1,37 +0,0 @@
# Runtime configuration of CKAN enabled through ckanext-envvars
# Information about how it works: https://github.com/okfn/ckanext-envvars
# Note that variables here take presedence over build/up time variables in .env
# General Settings
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000
CKAN_MAX_UPLOAD_SIZE_MB=10
# CKAN Plugins
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher harvest ckan_harvester
# CKAN requires storage path to be set in order for filestore to be enabled
CKAN__STORAGE_PATH=/srv/app/data
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
CKAN_SYSADMIN_NAME=sysadmin
CKAN_SYSADMIN_PASSWORD=password
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
# Email settings
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
# Datapusher configuration
CKAN__DATAPUSHER__URL=http://datapusher:8000
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
# Harvest settings
CKAN__HARVEST__MQ__TYPE=redis
CKAN__HARVEST__MQ__HOSTNAME=redis
# Solr configuration
CKAN_VERSION=2.9.11
CKAN_CORE_NAME=ckan

35
examples/harvest/.env
View File

@ -1,35 +0,0 @@
# Variables in this file will be used as build arguments when running
# docker-compose build and docker-compose up
# Verify correct substitution with "docker-compose config"
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
# docker-compose down -v
# docker-compose build
# docker-compose up -d
# Database
POSTGRES_PASSWORD=ckan
POSTGRES_PORT=5432
DATASTORE_READONLY_PASSWORD=datastore
# CKAN
CKAN_VERSION=2.9.11
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000
CKAN_MAX_UPLOAD_SIZE_MB=10
# Datapusher
DATAPUSHER_VERSION=0.0.17
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
DATAPUSHER_CHUNK_SIZE=16384
DATAPUSHER_CHUNK_INSERT_ROWS=250
DATAPUSHER_DOWNLOAD_TIMEOUT=30
DATAPUSHER_SSL_VERIFY=False
DATAPUSHER_REWRITE_RESOURCES=True
DATAPUSHER_REWRITE_URL=http://ckan:5000
# SOLR
CKAN_CORE_NAME=ckan
# Redis
REDIS_VERSION=6.0.7

58
examples/harvest/Dockerfile
View File

@ -1,58 +0,0 @@
###################
### Extensions ####
###################
FROM ghcr.io/keitaroinc/ckan:2.9.11 as extbuild
# Locations and tags, please use specific tags or revisions
ENV HARVEST_GIT_URL=https://github.com/ckan/ckanext-harvest
ENV HARVEST_GIT_BRANCH=v1.3.4
# Switch to the root user
USER root
# Install necessary packages to build extensions
RUN apk add --no-cache \
python3-dev \
gcc \
g++ \
libffi-dev \
openssl-dev \
rust \
cargo
# Fetch and build the custom CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${HARVEST_GIT_URL}@${HARVEST_GIT_BRANCH}#egg=ckanext-harvest
RUN pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${HARVEST_GIT_BRANCH}/pip-requirements.txt
RUN curl -o /wheels/harvest.txt https://raw.githubusercontent.com/ckan/ckanext-harvest/${HARVEST_GIT_BRANCH}/pip-requirements.txt
USER ckan
############
### MAIN ###
############
FROM ghcr.io/keitaroinc/ckan:2.9.11
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher harvest ckan_harvester
# Switch to the root user
USER root
COPY --from=extbuild /wheels /srv/app/ext_wheels
# Install and enable the custom extensions
RUN pip install --no-index --find-links=/srv/app/ext_wheels ckanext-harvest && \
pip install --no-index --find-links=/srv/app/ext_wheels -r /srv/app/ext_wheels/harvest.txt && \
# Configure plugins
ckan config-tool "${APP_DIR}/production.ini" "ckan.plugins = ${CKAN__PLUGINS}" && \
chown -R ckan:ckan /srv/app
# Remove wheels
RUN rm -rf /srv/app/ext_wheels
# Add harvest afterinit script
COPY ./afterinit.d/00_harvest.sh ${APP_DIR}/docker-afterinit.d/00_harvest.sh
# Switch to the ckan user
USER ckan

2
examples/harvest/afterinit.d/00_harvest.sh
View File

@ -1,2 +0,0 @@
#!/bin/sh
ckan -c /srv/app/production.ini harvester initdb

192
examples/harvest/docker-compose.yml
View File

@ -1,192 +0,0 @@
# docker-compose build && docker-compose up -d
version: "3"
volumes:
ckan_data:
pg_data:
solr_data:
services:
ckan:
container_name: ckan
build:
context: .
networks:
- frontend
- backend
depends_on:
- db
ports:
- "0.0.0.0:${CKAN_PORT}:5000"
env_file:
- ./.ckan-env
environment:
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
volumes:
- ckan_data:/srv/app/data
ckan-harvest-gather:
container_name: ckan-harvest-gather
build:
context: .
command: ckan -c /srv/app/production.ini harvester gather-consumer
restart: on-failure
networks:
- frontend
- backend
depends_on:
- db
- ckan
env_file:
- ./.ckan-env
environment:
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
ckan-harvest-fetch:
container_name: ckan-harvest-fetch
build:
context: .
command: ckan -c /srv/app/production.ini harvester fetch-consumer
restart: on-failure
networks:
- frontend
- backend
depends_on:
- db
- ckan
env_file:
- ./.ckan-env
environment:
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
ckan-harvest-run:
container_name: ckan-harvest-run
build:
context: .
command: /bin/sh -c "while true; do sleep 900; ckan -c /srv/app/production.ini harvester run; done"
networks:
- frontend
- backend
depends_on:
- db
- ckan
env_file:
- ./.ckan-env
environment:
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
ckan-harvest-cleanlog:
container_name: ckan-harvest-cleanlog
build:
context: .
command: /bin/sh -c "while true; do sleep 86400; ckan -c /srv/app/production.ini harvester clean_harvest_log; done"
networks:
- frontend
- backend
depends_on:
- db
- ckan
env_file:
- ./.ckan-env
environment:
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
datapusher:
container_name: datapusher
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
networks:
- frontend
- backend
ports:
- "8000:8000"
environment:
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
- DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
db:
container_name: db
build:
context: ../../compose/2.9
dockerfile: postgresql/Dockerfile
args:
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
networks:
- backend
environment:
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- PGDATA=/var/lib/postgresql/data/db
volumes:
- pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "ckan"]
solr:
container_name: solr
image: solr:8.11.1
networks:
- backend
env_file:
- ./.ckan-env
environment:
- CKAN_CORE_NAME=${CKAN_CORE_NAME}
- CKAN_VERSION=${CKAN_VERSION}
volumes:
- solr_data:/var/solr
- ${PWD}/../../compose/2.9/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
redis:
container_name: redis
image: redis:${REDIS_VERSION}
networks:
- backend
networks:
frontend:
backend:

42
examples/s3filestore/.ckan-env
View File

@ -1,42 +0,0 @@
# Runtime configuration of CKAN enabled through ckanext-envvars
# Information about how it works: https://github.com/okfn/ckanext-envvars
# Note that variables here take presedence over build/up time variables in .env
# General Settings
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000
CKAN_MAX_UPLOAD_SIZE_MB=10
# CKAN Plugins
CKAN__PLUGINS=envvars s3filestore image_view webpage_view text_view recline_view datastore datapusher
# CKAN requires storage path to be set in order for filestore to be enabled
CKAN__STORAGE_PATH=/srv/app/data
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
CKAN_SYSADMIN_NAME=sysadmin
CKAN_SYSADMIN_PASSWORD=password
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
CKAN__VIEWS__DEFAULT_VIEWS=image_view webpage_view text_view recline_view
# Email settings
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
# Datapusher configuration
CKAN__DATAPUSHER__URL=http://datapusher:8000
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
# S3/MINIO settings
CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=MINIOACCESSKEY
CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=MINIOSECRETKEY
CKANEXT__S3FILESTORE__AWS_BUCKET_NAME=ckan
CKANEXT__S3FILESTORE__HOST_NAME=http://minio:9000
CKANEXT__S3FILESTORE__REGION_NAME=us-east-1
CKANEXT__S3FILESTORE__SIGNATURE_VERSION=s3v4
# Solr configuration
CKAN_VERSION=2.9.11
CKAN_CORE_NAME=ckan

35
examples/s3filestore/.env
View File

@ -1,35 +0,0 @@
# Variables in this file will be used as build arguments when running
# docker-compose build and docker-compose up
# Verify correct substitution with "docker-compose config"
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
# docker-compose down -v
# docker-compose build
# docker-compose up -d
# Database
POSTGRES_PASSWORD=ckan
POSTGRES_PORT=5432
DATASTORE_READONLY_PASSWORD=datastore
# CKAN
CKAN_VERSION=2.9.11
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000
CKAN_MAX_UPLOAD_SIZE_MB=10
# Datapusher
DATAPUSHER_VERSION=0.0.17
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
DATAPUSHER_CHUNK_SIZE=16384
DATAPUSHER_CHUNK_INSERT_ROWS=250
DATAPUSHER_DOWNLOAD_TIMEOUT=30
DATAPUSHER_SSL_VERIFY=False
DATAPUSHER_REWRITE_RESOURCES=True
DATAPUSHER_REWRITE_URL=http://ckan:5000
# SOLR
CKAN_CORE_NAME=ckan
# Redis
REDIS_VERSION=6.0.7

43
examples/s3filestore/Dockerfile
View File

@ -1,43 +0,0 @@
###################
### Extensions ####
###################
FROM ghcr.io/keitaroinc/ckan:2.9.11 as extbuild
# Locations and tags, please use specific tags or revisions
ENV S3FILESTORE_GIT_URL=https://github.com/keitaroinc/ckanext-s3filestore
ENV S3FILESTORE_GIT_BRANCH=v1.0.0
# Switch to the root user
USER root
# Fetch and build the custom CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${S3FILESTORE_GIT_URL}@${S3FILESTORE_GIT_BRANCH}#egg=ckanext-s3filestore
RUN pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/keitaroinc/ckanext-s3filestore/${S3FILESTORE_GIT_BRANCH}/requirements.txt
RUN curl -o /wheels/s3filestore.txt https://raw.githubusercontent.com/keitaroinc/ckanext-s3filestore/${S3FILESTORE_GIT_BRANCH}/requirements.txt
USER ckan
############
### MAIN ###
############
FROM ghcr.io/keitaroinc/ckan:2.9.11
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
ENV CKAN__PLUGINS envvars s3filestore image_view webpage_view text_view recline_view datastore datapusher
# Switch to the root user
USER root
COPY --from=extbuild /wheels /srv/app/ext_wheels
# Install and enable the custom extensions
RUN pip install --no-index --find-links=/srv/app/ext_wheels ckanext-s3filestore && \
pip install --no-index --find-links=/srv/app/ext_wheels -r /srv/app/ext_wheels/s3filestore.txt && \
ckan config-tool "${APP_DIR}/production.ini" "ckan.plugins = ${CKAN__PLUGINS}" && \
chown -R ckan:ckan /srv/app
# Remove wheels
RUN rm -rf /srv/app/ext_wheels
USER ckan

122
examples/s3filestore/docker-compose.yml
View File

@ -1,122 +0,0 @@
# docker-compose build && docker-compose up -d
version: "3"
volumes:
minio_data:
pg_data:
solr_data:
services:
ckan:
container_name: ckan
build:
context: .
networks:
- frontend
- backend
depends_on:
- db
ports:
- "0.0.0.0:${CKAN_PORT}:5000"
env_file:
- ./.ckan-env
environment:
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
- CKAN_REDIS_URL=redis://redis:6379/1
- CKAN_SITE_URL=${CKAN_SITE_URL}
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
datapusher:
container_name: datapusher
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
networks:
- frontend
- backend
ports:
- "8000:8000"
environment:
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
- DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
db:
container_name: db
build:
context: ../../compose/2.9
dockerfile: postgresql/Dockerfile
args:
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
networks:
- backend
environment:
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- PGDATA=/var/lib/postgresql/data/db
volumes:
- pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "ckan"]
solr:
container_name: solr
image: solr:8.11.1
networks:
- backend
env_file:
- ./.ckan-env
environment:
- CKAN_CORE_NAME=${CKAN_CORE_NAME}
- CKAN_VERSION=${CKAN_VERSION}
volumes:
- solr_data:/var/solr
- ${PWD}/../../compose/2.9/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
redis:
container_name: redis
image: redis:${REDIS_VERSION}
networks:
- backend
minio:
container_name: minio
image: minio/minio:RELEASE.2020-08-08T04-50-06Z
networks:
- backend
- frontend
ports:
- "0.0.0.0:9000:9000"
environment:
- MINIO_ACCESS_KEY=MINIOACCESSKEY
- MINIO_SECRET_KEY=MINIOSECRETKEY
volumes:
- minio_data:/data
command: server /data
mc:
container_name: mc
image: minio/mc:RELEASE.2020-08-08T02-33-58Z
networks:
- backend
depends_on:
- minio
entrypoint: >
/bin/sh -c "
/usr/bin/mc config host rm local;
/usr/bin/mc config host add --api s3v4 local http://minio:9000 MINIOACCESSKEY MINIOSECRETKEY;
/usr/bin/mc mb local/ckan/;
/usr/bin/mc policy set download local/ckan/storage;
"
networks:
frontend:
backend:

196
images/ckan/2.7/Dockerfile
View File

@ -1,196 +0,0 @@
##################
### Build CKAN ###
##################
FROM alpine:3.14.2 as ckanbuild
# Used by Github Actions to tag the image with
ENV IMAGE_TAG=2.7.12
# Set CKAN version to build
ENV GIT_URL=https://github.com/ckan/ckan.git
ENV GIT_BRANCH=ckan-2.7.12
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
WORKDIR ${SRC_DIR}
# Packages to build CKAN requirements and plugins
RUN apk add --no-cache \
git \
curl \
python2 \
postgresql-dev \
linux-headers \
gcc \
make \
g++ \
autoconf \
automake \
libtool \
patch \
musl-dev \
pcre-dev \
pcre \
python2-dev \
libffi-dev \
libxml2-dev \
libxslt-dev
# Install version 9.x of postgresql-dev so that CKAN requirements can be built
RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.6/main \
postgresql-dev~=9.6
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==21.12.0 greenlet==1.1.3
###########################
### Default-Extensions ####
###########################
FROM alpine:3.14.2 as extbuild
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
# List of default extensions
ENV DEFAULT_EXTENSIONS envvars
# Locations and tags, please use specific tags or revisions
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
ENV ENVVARS_GIT_BRANCH=0.0.1
RUN apk add --no-cache \
git \
curl \
python2 \
python2-dev
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Fetch and build the default CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
############
### MAIN ###
############
FROM alpine:3.14.2
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_DIR=${SRC_DIR}/ckan
ENV DATA_DIR=/srv/app/data
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
# Install necessary packages to run CKAN
RUN apk add --no-cache \
git \
bash \
git \
gettext \
curl \
python2 \
libmagic \
pcre \
libxslt \
libxml2 \
tzdata \
apache2-utils && \
# Install version 9.x of postgresql-client so that CKAN can run
apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.6/main \
postgresql-client~=9.6 && \
# Create SRC_DIR
mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Get artifacts from build stages
COPY --from=ckanbuild /wheels /srv/app/wheels
COPY --from=extbuild /wheels /srv/app/ext_wheels
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
# Additional install steps for build stages artifacts
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi==2.0.19.1 gevent==21.12.0
# Create a local user and group to run the app
RUN addgroup -g 92 -S ckan && \
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
WORKDIR ${CKAN_DIR}
# Install CKAN
RUN pip install -e /srv/app/src/ckan && \
cp who.ini ${APP_DIR} && \
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
# Install default CKAN extensions
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
# Create and update CKAN config
# Set timezone
echo "UTC" > /etc/timezone && \
# Generate CKAN config
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Set the default level for extensions to INFO
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
# Create the data directory
mkdir ${DATA_DIR} && \
# Change ownership to app user
chown -R ckan:ckan /srv/app
# Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
# Install python2 secrets for generating sessions
RUN pip install python2-secrets
# Copy necessary scripts
COPY setup/app ${APP_DIR}
WORKDIR ${APP_DIR}
# Copy the alias script for paster to be ckan so it's compatible with 2.9
COPY setup/bin/ckan /usr/bin/ckan
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
USER ckan
CMD ["/srv/app/start_ckan.sh"]

240
images/ckan/2.7/Dockerfile.focal
View File

@ -1,240 +0,0 @@
##################
### Build CKAN ###
##################
FROM ubuntu:focal-20210827 as ckanbuild
# Used by Github Actions to tag the image with
ENV IMAGE_TAG=2.7.12-focal
# Set CKAN version to build
ENV GIT_URL=https://github.com/ckan/ckan.git
ENV GIT_BRANCH=ckan-2.7.12
# Set timezone
ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Set Locale
ENV LC_ALL=en_US.UTF-8
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
WORKDIR ${SRC_DIR}
# Set the locale
RUN apt-get update
RUN apt-get install --no-install-recommends -y locales
RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
RUN dpkg-reconfigure --frontend=noninteractive locales
RUN update-locale LANG=${LC_ALL}
# Instal apt-utils
RUN apt-get install --no-install-recommends -y \
apt-utils
# Packages to build CKAN requirements and plugins
RUN apt-get install --no-install-recommends -y \
git \
gnupg \
curl \
ca-certificates \
python \
linux-headers-generic \
gcc-10 \
make \
g++-10 \
autoconf \
automake \
libtool \
patch \
libpcre3-dev \
libpcre3 \
python-dev \
libffi-dev \
libxml2-dev \
libxslt-dev
# Install version 9.x of postgresql-dev so that CKAN requirements can be built
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" >> /etc/apt/sources.list.d/postgresql.list'
RUN apt-get update
RUN apt-get install --no-install-recommends -y \
postgresql-server-dev-9.6
# Use gcc 10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10
#postgresql-server-dev-9.6 Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==21.12.0 greenlet==1.1.3
###########################
### Default-Extensions ####
###########################
FROM ubuntu:focal-20210827 as extbuild
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
# List of default extensions
ENV DEFAULT_EXTENSIONS envvars
# Locations and tags, please use specific tags or revisions
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
ENV ENVVARS_GIT_BRANCH=0.0.1
RUN apt-get update && \
apt-get install --no-install-recommends -y \
git \
curl \
ca-certificates \
python \
python-dev
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Fetch and build the default CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
############
### MAIN ###
############
FROM ubuntu:focal-20210827
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
# Set timezone
ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Set Locale
ENV LC_ALL=en_US.UTF-8
# Set the locale
RUN apt-get update && \
apt-get install --no-install-recommends -y locales && \
sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \
dpkg-reconfigure --frontend=noninteractive locales && \
update-locale LANG=${LC_ALL} && \
rm -rf /var/lib/apt/lists/*
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_DIR=${SRC_DIR}/ckan
ENV DATA_DIR=/srv/app/data
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
# Install necessary packages to run CKAN
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update && \
apt-get install --no-install-recommends -y \
gettext \
curl \
ca-certificates \
git \
gnupg \
python \
libpython2.7 \
libmagic1 \
libpcre3 \
libxslt1.1 \
libxml2 \
tzdata \
apache2-utils && \
rm -rf /var/lib/apt/lists/* && \
# Install version 9.x of postgresql-client so that CKAN can run
curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" >> /etc/apt/sources.list.d/postgresql.list' && \
apt-get update && \
apt-get install --no-install-recommends -y \
postgresql-client-9.6 && \
rm -rf /var/lib/apt/lists/* && \
# Create SRC_DIR
mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Get artifacts from build stages
COPY --from=ckanbuild /wheels /srv/app/wheels
COPY --from=extbuild /wheels /srv/app/ext_wheels
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
# Additional install steps for build stages artifacts
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi==2.0.19.1 gevent==21.12.0
# Create a local user and group to run the app
RUN groupadd -g 92 ckan && \
useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan
WORKDIR ${CKAN_DIR}
# Install CKAN
RUN pip install -e /srv/app/src/ckan && \
cp who.ini ${APP_DIR} && \
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
# Install default CKAN extensions
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
# Create and update CKAN config
# Generate CKAN config
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Set the default level for extensions to INFO
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
# Create the data directory
mkdir ${DATA_DIR} && \
# Change ownership to app user
chown -R ckan:ckan /srv/app
# Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
# Copy necessary scripts
COPY setup/app ${APP_DIR}
WORKDIR ${APP_DIR}
# Copy the alias script for paster to be ckan so it's compatible with 2.9
COPY setup/bin/ckan /usr/bin/ckan
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
USER ckan
CMD ["/srv/app/start_ckan.sh"]

0
images/ckan/2.7/patches/.gitkeep
View File

5
images/ckan/2.7/scripts/apply_ckan_patches.sh
View File

@ -1,5 +0,0 @@
#!/bin/bash
shopt -s nullglob
for patch in patches/*.patch; do
/usr/bin/patch -p0 -i $patch
done

4
images/ckan/2.7/setup/app/extra_scripts.sh
View File

@ -1,4 +0,0 @@
#!/bin/bash
# this is called before uwsgi is executed
# uset his to add extra scripts before ckan is started

25
images/ckan/2.7/setup/app/maintenance/api/3/action/status_show/index.html
View File

@ -1,25 +0,0 @@
<!--
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<title>Maintenance</title>
</head>
<body>
<h1>Maintenance</h1>
<p>Our data portal is currently in maintenance, please try in a while.</p>
</body>
</html>

25
images/ckan/2.7/setup/app/maintenance/index.html
View File

@ -1,25 +0,0 @@
<!--
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<title>Maintenance</title>
</head>
<body>
<h1>Maintenance</h1>
<p>Our data portal is currently in maintenance, please try in a while.</p>
</body>
</html>

38
images/ckan/2.7/setup/app/maintenance/serve.py
View File

@ -1,38 +0,0 @@
"""
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from SimpleHTTPServer import SimpleHTTPRequestHandler
from BaseHTTPServer import HTTPServer
from SocketServer import ThreadingMixIn
import os
PORT = 5000
web_dir = os.path.join(os.path.dirname(__file__))
os.chdir(web_dir)
Handler = SimpleHTTPRequestHandler
class MaintenanceServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
if __name__ == "__main__":
httpd = MaintenanceServer(("0.0.0.0", PORT), Handler)
print("Starting maintenance mode")
httpd.serve_forever()

200
images/ckan/2.7/setup/app/prerun.py
View File

@ -1,200 +0,0 @@
"""
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import sys
import subprocess
import psycopg2
import urllib2
import re
import time
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
RETRY = 5
def check_db_connection(retry=None):
print('[prerun] Start check_db_connection...')
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
try:
connection = psycopg2.connect(conn_str)
except psycopg2.Error as e:
print((str(e)))
print('[prerun] Unable to connect to the database...try again in a while.')
import time
time.sleep(10)
check_db_connection(retry = retry - 1)
else:
connection.close()
def check_solr_connection(retry=None):
print('[prerun] Start check_solr_connection...')
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
url = os.environ.get('CKAN_SOLR_URL', '')
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
try:
connection = urllib2.urlopen(search_url)
except urllib2.URLError as e:
print('[prerun] Unable to connect to solr...try again in a while.')
import time
time.sleep(10)
check_solr_connection(retry = retry - 1)
else:
import re
conn_info = connection.read()
# SolrCloud
conn_info = re.sub(r'"zkConnected":true', '"zkConnected":True', conn_info)
eval(conn_info)
def init_db():
print('[prerun] Start init_db...')
db_command = ['paster', '--plugin=ckan', 'db', 'init', '-c', ckan_ini]
print('[prerun] Initializing or upgrading db - start using paster db init')
try:
# run init scripts
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
print('[prerun] Initializing or upgrading db - end')
except subprocess.CalledProcessError as e:
if 'OperationalError' in e.output:
print('[prerun] Database not ready, waiting a bit before exit...')
import time
time.sleep(5)
sys.exit(1)
else:
print((e.output))
raise e
print('[prerun] Initializing or upgrading db - finish')
def init_datastore():
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print('[prerun] Skipping datastore initialization')
return
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
'set-permissions', '-c', ckan_ini]
connection = psycopg2.connect(conn_str)
cursor = connection.cursor()
print('[prerun] Initializing datastore db - start')
try:
datastore_perms = subprocess.Popen(
datastore_perms_command,
stdout=subprocess.PIPE)
perms_sql = datastore_perms.stdout.read()
# Remove internal pg command as psycopg2 does not like it
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql.decode('utf-8'))
cursor.execute(perms_sql)
for notice in connection.notices:
print(notice)
connection.commit()
print('[prerun] Initializing datastore db - end')
print((datastore_perms.stdout.read()))
except psycopg2.Error as e:
print('[prerun] Could not initialize datastore')
print((str(e)))
except subprocess.CalledProcessError as e:
if 'OperationalError' in e.output:
print((e.output))
print('[prerun] Database not ready, waiting a bit before exit...')
time.sleep(5)
sys.exit(1)
else:
print((e.output))
raise e
finally:
cursor.close()
connection.close()
def create_sysadmin():
print('[prerun] Start create_sysadmin...')
name = os.environ.get('CKAN_SYSADMIN_NAME')
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
if name and password and email:
# Check if user exists
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
out = subprocess.check_output(command)
if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')):
print('[prerun] Sysadmin user exists, skipping creation')
return
# Create user
command = ['paster', '--plugin=ckan', 'user', 'add',
name,
'password=' + password,
'email=' + email,
'-c', ckan_ini]
subprocess.call(command)
print(('[prerun] Created user {0}'.format(name)))
# Make it sysadmin
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
name,
'-c', ckan_ini]
subprocess.call(command)
print(('[prerun] Made user {0} a sysadmin'.format(name)))
if __name__ == '__main__':
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
if maintenance:
print('[prerun] Maintenance mode, skipping setup...')
else:
check_db_connection()
check_solr_connection()
init_db()
if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
init_datastore()
create_sysadmin()

74
images/ckan/2.7/setup/app/start_ckan.sh
View File

@ -1,74 +0,0 @@
#!/bin/bash
# Run any startup scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-entrypoint.d" ]]
then
for f in ${APP_DIR}/docker-entrypoint.d/*; do
case "$f" in
*.sh) echo "$0: Running init file $f"; . "$f" ;;
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Add session secret from chart
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET ]];then
echo "Not all environment variables are set. Generating sessions..."
else
echo "Setting session secrets from environment variables"
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
fi
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
then
echo "Setting beaker.session.secret in ini file"
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python -c 'import secrets; print(secrets.token_urlsafe())')"
fi
# Set the common uwsgi options
echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger /srv/app/production.ini --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch"
# Run the prerun script to init CKAN and create the default admin user
python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }
# Check if we are in maintenance mode and if yes serve the maintenance pages
if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi
# Run any after prerun/init scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
then
for f in ${APP_DIR}/docker-afterinit.d/*; do
case "$f" in
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then
if [ "$PASSWORD_PROTECT" = true ]
then
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
then
# Generate htpasswd file for basicauth
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
# Start uwsgi with basicauth
uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
else
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
exit 1
fi
else
# Start uwsgi
uwsgi $UWSGI_OPTS
fi
else
echo "[prerun] failed...not starting CKAN."
fi

2
images/ckan/2.7/setup/app/uwsgi.conf
View File

@ -1,2 +0,0 @@
[uwsgi]
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd

15
images/ckan/2.7/setup/bin/ckan
View File

@ -1,15 +0,0 @@
#!/bin/bash
# CKAN <2.9 uses paster and the ckan plugin for paster which expects the configuration to be passed
# to a subcommand, example: paster --plugin=ckan datastore -c /srv/app/production.ini set-permissions
# CKAN >2.9 ckan CLI command expects the configuration option as the second and third parameter
# Example: ckan -c /srv/app/production.ini datastore set-permissions
#
# Shift three arguments so that we can reorder the argument list
# ckan -c /srv/app/production.ini datastore set-permissions
# becomes
# paster --plugin=ckan datastore -c /srv/app/production.ini set-permissions
config_switch=$1
config_file=$2
subcommand=$3
shift 3
paster --plugin=ckan "$subcommand" "$config_switch" "$config_file" "$@"

189
images/ckan/2.8/Dockerfile
View File

@ -1,189 +0,0 @@
##################
### Build CKAN ###
##################
FROM alpine:3.14.2 as ckanbuild
# Used by Github Actions to tag the image with
ENV IMAGE_TAG=2.8.12
# Set CKAN version to build
ENV GIT_URL=https://github.com/ckan/ckan.git
ENV GIT_BRANCH=ckan-2.8.12
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
WORKDIR ${SRC_DIR}
# Packages to build CKAN requirements and plugins
RUN apk add --no-cache \
git \
curl \
python2 \
postgresql-dev \
linux-headers \
gcc \
make \
g++ \
autoconf \
automake \
patch \
libtool \
musl-dev \
pcre-dev \
pcre \
python2-dev \
libffi-dev \
libxml2-dev \
libxslt-dev
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==21.12.0 greenlet==1.1.3
###########################
### Default-Extensions ####
###########################
FROM alpine:3.14.2 as extbuild
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
# List of default extensions
ENV DEFAULT_EXTENSIONS envvars
# Locations and tags, please use specific tags or revisions
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
ENV ENVVARS_GIT_BRANCH=0.0.1
RUN apk add --no-cache \
git \
curl \
python2 \
python2-dev
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Fetch and build the default CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
############
### MAIN ###
############
FROM alpine:3.14.2
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_DIR=${SRC_DIR}/ckan
ENV DATA_DIR=/srv/app/data
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
# Install necessary packages to run CKAN
RUN apk add --no-cache git \
bash \
git \
gettext \
curl \
postgresql-client \
python2 \
libmagic \
pcre \
libxslt \
libxml2 \
tzdata \
apache2-utils && \
# Create SRC_DIR
mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Get artifacts from build stages
COPY --from=ckanbuild /wheels /srv/app/wheels
COPY --from=extbuild /wheels /srv/app/ext_wheels
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
# Additional install steps for build stages artifacts
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi==2.0.19.1 gevent==21.12.0
# Create a local user and group to run the app
RUN addgroup -g 92 -S ckan && \
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
WORKDIR ${CKAN_DIR}
# Install CKAN
RUN pip install -e /srv/app/src/ckan && \
cp who.ini ${APP_DIR} && \
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
# Install default CKAN extensions
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
# Create and update CKAN config
# Set timezone
echo "UTC" > /etc/timezone && \
# Generate CKAN config
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Set the default level for extensions to INFO
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
# Create the data directory
mkdir ${DATA_DIR} && \
# Change ownership to app user
chown -R ckan:ckan /srv/app
# Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
# Install python2 secrets for generating sessions
RUN pip install python2-secrets
# Copy necessary scripts
COPY setup/app ${APP_DIR}
WORKDIR ${APP_DIR}
# Copy the alias script for paster to be ckan so it's compatible with 2.9
COPY setup/bin/ckan /usr/bin/ckan
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
USER ckan
CMD ["/srv/app/start_ckan.sh"]

226
images/ckan/2.8/Dockerfile.focal
View File

@ -1,226 +0,0 @@
##################
### Build CKAN ###
##################
FROM ubuntu:focal-20210827 as ckanbuild
# Used by Github Actions to tag the image with
ENV IMAGE_TAG=2.8.12-focal
# Set CKAN version to build
ENV GIT_URL=https://github.com/ckan/ckan.git
ENV GIT_BRANCH=ckan-2.8.12
# Set timezone
ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Set Locale
ENV LC_ALL=en_US.UTF-8
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
WORKDIR ${SRC_DIR}
# Set the locale
RUN apt-get update
RUN apt-get install --no-install-recommends -y locales
RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
RUN dpkg-reconfigure --frontend=noninteractive locales
RUN update-locale LANG=${LC_ALL}
# Instal apt-utils
RUN apt-get install --no-install-recommends -y \
apt-utils
# Packages to build CKAN requirements and plugins
RUN apt-get install --no-install-recommends -y \
git \
curl \
ca-certificates \
python \
libpq-dev \
linux-headers-generic \
gcc-10 \
make \
g++-10 \
autoconf \
automake \
libtool \
patch \
libpcre3-dev \
libpcre3 \
python-dev \
libffi-dev \
libxml2-dev \
libxslt-dev
# Use gcc 10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==21.12.0 greenlet==1.1.3
###########################
### Default-Extensions ####
###########################
FROM ubuntu:focal-20210827 as extbuild
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
# List of default extensions
ENV DEFAULT_EXTENSIONS envvars
# Locations and tags, please use specific tags or revisions
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
ENV ENVVARS_GIT_BRANCH=0.0.1
RUN apt-get update && \
apt-get install --no-install-recommends -y \
git \
curl \
ca-certificates \
python \
python-dev
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Fetch and build the default CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
############
### MAIN ###
############
FROM ubuntu:focal-20210827
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
# Set timezone
ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Set Locale
ENV LC_ALL=en_US.UTF-8
# Set the locale
RUN apt-get update && \
apt-get install --no-install-recommends -y locales && \
sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \
dpkg-reconfigure --frontend=noninteractive locales && \
update-locale LANG=${LC_ALL} && \
rm -rf /var/lib/apt/lists/*
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_DIR=${SRC_DIR}/ckan
ENV DATA_DIR=/srv/app/data
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
# Install necessary packages to run CKAN
RUN apt-get update && \
apt-get install --no-install-recommends -y \
gettext \
curl \
ca-certificates \
git \
libpq5 \
postgresql-client \
python \
libpython2.7 \
libmagic1 \
libpcre3 \
libxslt1.1 \
libxml2 \
tzdata \
apache2-utils && \
rm -rf /var/lib/apt/lists/* && \
# Create SRC_DIR
mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Get artifacts from build stages
COPY --from=ckanbuild /wheels /srv/app/wheels
COPY --from=extbuild /wheels /srv/app/ext_wheels
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
# Additional install steps for build stages artifacts
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi==2.0.19.1 gevent==21.12.0
# Create a local user and group to run the app
RUN groupadd -g 92 ckan && \
useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan
WORKDIR ${CKAN_DIR}
# Install CKAN
RUN pip install -e /srv/app/src/ckan && \
cp who.ini ${APP_DIR} && \
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
# Install default CKAN extensions
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
# Create and update CKAN config
# Generate CKAN config
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Set the default level for extensions to INFO
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
# Create the data directory
mkdir ${DATA_DIR} && \
# Change ownership to app user
chown -R ckan:ckan /srv/app
# Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
# Copy necessary scripts
COPY setup/app ${APP_DIR}
WORKDIR ${APP_DIR}
# Copy the alias script for paster to be ckan so it's compatible with 2.9
COPY setup/bin/ckan /usr/bin/ckan
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
USER ckan
CMD ["/srv/app/start_ckan.sh"]

0
images/ckan/2.8/patches/.gitkeep
View File

5
images/ckan/2.8/scripts/apply_ckan_patches.sh
View File

@ -1,5 +0,0 @@
#!/bin/bash
shopt -s nullglob
for patch in patches/*.patch; do
/usr/bin/patch -p0 -i $patch
done

4
images/ckan/2.8/setup/app/extra_scripts.sh
View File

@ -1,4 +0,0 @@
#!/bin/bash
# this is called before uwsgi is executed
# uset his to add extra scripts before ckan is started

25
images/ckan/2.8/setup/app/maintenance/api/3/action/status_show/index.html
View File

@ -1,25 +0,0 @@
<!--
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<title>Maintenance</title>
</head>
<body>
<h1>Maintenance</h1>
<p>Our data portal is currently in maintenance, please try in a while.</p>
</body>
</html>

25
images/ckan/2.8/setup/app/maintenance/index.html
View File

@ -1,25 +0,0 @@
<!--
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<title>Maintenance</title>
</head>
<body>
<h1>Maintenance</h1>
<p>Our data portal is currently in maintenance, please try in a while.</p>
</body>
</html>

38
images/ckan/2.8/setup/app/maintenance/serve.py
View File

@ -1,38 +0,0 @@
"""
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from SimpleHTTPServer import SimpleHTTPRequestHandler
from BaseHTTPServer import HTTPServer
from SocketServer import ThreadingMixIn
import os
PORT = 5000
web_dir = os.path.join(os.path.dirname(__file__))
os.chdir(web_dir)
Handler = SimpleHTTPRequestHandler
class MaintenanceServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
if __name__ == "__main__":
httpd = MaintenanceServer(("0.0.0.0", PORT), Handler)
print("Starting maintenance mode")
httpd.serve_forever()

200
images/ckan/2.8/setup/app/prerun.py
View File

@ -1,200 +0,0 @@
"""
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import sys
import subprocess
import psycopg2
import urllib2
import re
import time
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
RETRY = 5
def check_db_connection(retry=None):
print('[prerun] Start check_db_connection...')
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
try:
connection = psycopg2.connect(conn_str)
except psycopg2.Error as e:
print((str(e)))
print('[prerun] Unable to connect to the database...try again in a while.')
import time
time.sleep(10)
check_db_connection(retry = retry - 1)
else:
connection.close()
def check_solr_connection(retry=None):
print('[prerun] Start check_solr_connection...')
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
url = os.environ.get('CKAN_SOLR_URL', '')
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
try:
connection = urllib2.urlopen(search_url)
except urllib2.URLError as e:
print('[prerun] Unable to connect to solr...try again in a while.')
import time
time.sleep(10)
check_solr_connection(retry = retry - 1)
else:
import re
conn_info = connection.read()
# SolrCloud
conn_info = re.sub(r'"zkConnected":true', '"zkConnected":True', conn_info)
eval(conn_info)
def init_db():
print('[prerun] Start init_db...')
db_command = ['paster', '--plugin=ckan', 'db', 'init', '-c', ckan_ini]
print('[prerun] Initializing or upgrading db - start using paster db init')
try:
# run init scripts
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
print('[prerun] Initializing or upgrading db - end')
except subprocess.CalledProcessError as e:
if 'OperationalError' in e.output:
print('[prerun] Database not ready, waiting a bit before exit...')
import time
time.sleep(5)
sys.exit(1)
else:
print((e.output))
raise e
print('[prerun] Initializing or upgrading db - finish')
def init_datastore():
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print('[prerun] Skipping datastore initialization')
return
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
'set-permissions', '-c', ckan_ini]
connection = psycopg2.connect(conn_str)
cursor = connection.cursor()
print('[prerun] Initializing datastore db - start')
try:
datastore_perms = subprocess.Popen(
datastore_perms_command,
stdout=subprocess.PIPE)
perms_sql = datastore_perms.stdout.read()
# Remove internal pg command as psycopg2 does not like it
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql.decode('utf-8'))
cursor.execute(perms_sql)
for notice in connection.notices:
print(notice)
connection.commit()
print('[prerun] Initializing datastore db - end')
print((datastore_perms.stdout.read()))
except psycopg2.Error as e:
print('[prerun] Could not initialize datastore')
print((str(e)))
except subprocess.CalledProcessError as e:
if 'OperationalError' in e.output:
print((e.output))
print('[prerun] Database not ready, waiting a bit before exit...')
time.sleep(5)
sys.exit(1)
else:
print((e.output))
raise e
finally:
cursor.close()
connection.close()
def create_sysadmin():
print('[prerun] Start create_sysadmin...')
name = os.environ.get('CKAN_SYSADMIN_NAME')
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
if name and password and email:
# Check if user exists
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
out = subprocess.check_output(command)
if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')):
print('[prerun] Sysadmin user exists, skipping creation')
return
# Create user
command = ['paster', '--plugin=ckan', 'user', 'add',
name,
'password=' + password,
'email=' + email,
'-c', ckan_ini]
subprocess.call(command)
print(('[prerun] Created user {0}'.format(name)))
# Make it sysadmin
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
name,
'-c', ckan_ini]
subprocess.call(command)
print(('[prerun] Made user {0} a sysadmin'.format(name)))
if __name__ == '__main__':
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
if maintenance:
print('[prerun] Maintenance mode, skipping setup...')
else:
check_db_connection()
check_solr_connection()
init_db()
if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
init_datastore()
create_sysadmin()

74
images/ckan/2.8/setup/app/start_ckan.sh
View File

@ -1,74 +0,0 @@
#!/bin/bash
# Run any startup scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-entrypoint.d" ]]
then
for f in ${APP_DIR}/docker-entrypoint.d/*; do
case "$f" in
*.sh) echo "$0: Running init file $f"; . "$f" ;;
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Add session secret from chart
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET ]];then
echo "Not all environment variables are set. Generating sessions..."
else
echo "Setting session secrets from environment variables"
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
fi
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
then
echo "Setting beaker.session.secret in ini file"
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python -c 'import secrets; print(secrets.token_urlsafe())')"
fi
# Set the common uwsgi options
echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger /srv/app/production.ini --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch"
# Run the prerun script to init CKAN and create the default admin user
python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }
# Check if we are in maintenance mode and if yes serve the maintenance pages
if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi
# Run any after prerun/init scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
then
for f in ${APP_DIR}/docker-afterinit.d/*; do
case "$f" in
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then
if [ "$PASSWORD_PROTECT" = true ]
then
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
then
# Generate htpasswd file for basicauth
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
# Start uwsgi with basicauth
uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
else
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
exit 1
fi
else
# Start uwsgi
uwsgi $UWSGI_OPTS
fi
else
echo "[prerun] failed...not starting CKAN."
fi

2
images/ckan/2.8/setup/app/uwsgi.conf
View File

@ -1,2 +0,0 @@
[uwsgi]
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd

15
images/ckan/2.8/setup/bin/ckan
View File

@ -1,15 +0,0 @@
#!/bin/bash
# CKAN <2.9 uses paster and the ckan plugin for paster which expects the configuration to be passed
# to a subcommand, example: paster --plugin=ckan datastore -c /srv/app/production.ini set-permissions
# CKAN >2.9 ckan CLI command expects the configuration option as the second and third parameter
# Example: ckan -c /srv/app/production.ini datastore set-permissions
#
# Shift three arguments so that we can reorder the argument list
# ckan -c /srv/app/production.ini datastore set-permissions
# becomes
# paster --plugin=ckan datastore -c /srv/app/production.ini set-permissions
config_switch=$1
config_file=$2
subcommand=$3
shift 3
paster --plugin=ckan "$subcommand" "$config_switch" "$config_file" "$@"

203
images/ckan/2.9/Dockerfile
View File

@ -1,203 +0,0 @@
##################
### Build CKAN ###
##################
FROM alpine:3.13.7 as ckanbuild
# Used by Github Actions to tag the image with
ENV IMAGE_TAG=2.9.11
# Set CKAN version to build
ENV GIT_URL=https://github.com/ckan/ckan.git
ENV GIT_BRANCH=ckan-2.9.11
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
WORKDIR ${SRC_DIR}
# Packages to build CKAN requirements and plugins
RUN apk add --no-cache \
python3 \
python3-dev \
git \
curl \
postgresql-dev \
linux-headers \
gcc \
make \
g++ \
autoconf \
automake \
libtool \
patch \
musl-dev \
pcre-dev \
pcre \
libffi-dev \
libxml2-dev \
libxslt-dev
# Link python to python3
RUN ln -s /usr/bin/python3 /usr/bin/python
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
python ${SRC_DIR}/get-pip.py
# Downgrade setuptools so that CKAN requirements can be built
RUN pip install setuptools==44.1.0
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN cd ${SRC_DIR} && ls -lah ${SRC_DIR} && ash ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736
RUN echo 'Cython < 3.0' > /tmp/constraint.txt
RUN PIP_CONSTRAINT=/tmp/constraint.txt pip wheel --wheel-dir=/wheels PyYAML==5.4.1
# RUN pip-compile ckan/requirements.in
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3
###########################
### Default-Extensions ####
###########################
FROM alpine:3.13.7 as extbuild
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
# List of default extensions
ENV DEFAULT_EXTENSIONS envvars
# Locations and tags, please use specific tags or revisions
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
ENV ENVVARS_GIT_BRANCH=0.0.1
RUN apk add --no-cache \
python3 \
python3-dev \
git \
curl
# Link python to python3
RUN ln -s /usr/bin/python3 /usr/bin/python
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
python ${SRC_DIR}/get-pip.py
# Fetch and build the default CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
############
### MAIN ###
############
FROM alpine:3.13.7
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_DIR=${SRC_DIR}/ckan
ENV DATA_DIR=/srv/app/data
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
# Install necessary packages to run CKAN
RUN apk add --no-cache \
python3 \
bash \
git \
gettext \
curl \
postgresql-client \
libmagic \
pcre \
libxslt \
libxml2 \
tzdata \
apache2-utils && \
# Create SRC_DIR
mkdir -p ${SRC_DIR} && \
# Link python to python3
ln -s /usr/bin/python3 /usr/bin/python
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
python ${SRC_DIR}/get-pip.py
# Get artifacts from build stages
COPY --from=ckanbuild /wheels /srv/app/wheels
COPY --from=extbuild /wheels /srv/app/ext_wheels
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
# Additional install steps for build stages artifacts
RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0
# Create a local user and group to run the app
RUN addgroup -g 92 -S ckan && \
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
WORKDIR ${CKAN_DIR}
# Install CKAN
RUN pip install -e /srv/app/src/ckan && \
cp who.ini ${APP_DIR} && \
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
# Install default CKAN extensions
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
# Create and update CKAN config
# Set timezone
echo "UTC" > /etc/timezone && \
# Generate CKAN config
ckan generate config ${APP_DIR}/production.ini && \
ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
# Configure plugins
ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Create the data directory
mkdir ${DATA_DIR} && \
# Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created
ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \
# Set the default level for extensions to INFO
ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
# Change ownership to app user
chown -R ckan:ckan /srv/app
RUN pip install sqlalchemy==1.3.19
# Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
# Copy necessary scripts
COPY setup/app ${APP_DIR}
WORKDIR ${APP_DIR}
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
USER ckan
CMD ["/srv/app/start_ckan.sh"]

244
images/ckan/2.9/Dockerfile.focal
View File

@ -1,244 +0,0 @@
##################
### Build CKAN ###
##################
FROM ubuntu:focal-20210827 as ckanbuild
# Used by Github Actions to tag the image with
ENV IMAGE_TAG=2.9.11-focal
# Set CKAN version to build
ENV GIT_URL=https://github.com/ckan/ckan.git
ENV GIT_BRANCH=ckan-2.9.11
# Set timezone
ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Set Locale
ENV LC_ALL=en_US.UTF-8
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
WORKDIR ${SRC_DIR}
# Set the locale
RUN apt-get update
RUN apt-get install --no-install-recommends -y locales
RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
RUN dpkg-reconfigure --frontend=noninteractive locales
RUN update-locale LANG=${LC_ALL}
# Instal apt-utils
RUN apt-get install --no-install-recommends -y \
apt-utils
# Packages to build CKAN requirements and plugins
RUN apt-get install --no-install-recommends -y \
git \
curl \
ca-certificates \
python3 \
libpq-dev \
linux-headers-generic \
gcc-10 \
make \
g++-10 \
autoconf \
automake \
libtool \
patch \
libpcre3-dev \
libpcre3 \
python3-dev \
libffi-dev \
libxml2-dev \
libxslt-dev
# Use gcc 10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10
# Link python to python3
RUN ln -s /usr/bin/python3 /usr/bin/python
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
python ${SRC_DIR}/get-pip.py
# Downgrade setuptools so that CKAN requirements can be built
RUN pip install setuptools==44.1.0
# Fetch and build CKAN and requirements
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Copy patches and apply patches script
COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3
###########################
### Default-Extensions ####
###########################
FROM ubuntu:focal-20210827 as extbuild
# Set src dirs
ENV SRC_DIR=/srv/app/src
ENV PIP_SRC=${SRC_DIR}
# List of default extensions
ENV DEFAULT_EXTENSIONS envvars
# Locations and tags, please use specific tags or revisions
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
ENV ENVVARS_GIT_BRANCH=0.0.1
RUN apt-get update && \
apt-get install --no-install-recommends -y \
git \
curl \
ca-certificates \
python3 \
python3-dev
# Link python to python3
RUN ln -s /usr/bin/python3 /usr/bin/python
# Create the src directory
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
python ${SRC_DIR}/get-pip.py
# Downgrade setuptools so that CKAN requirements can be built
RUN pip install setuptools==44.1.0
# Fetch and build the default CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
############
### MAIN ###
############
FROM ubuntu:focal-20210827
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
# Set timezone
ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Set Locale
ENV LC_ALL=en_US.UTF-8
# Set the locale
RUN apt-get update && \
apt-get install --no-install-recommends -y locales && \
sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \
dpkg-reconfigure --frontend=noninteractive locales && \
update-locale LANG=${LC_ALL} && \
rm -rf /var/lib/apt/lists/*
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_DIR=${SRC_DIR}/ckan
ENV DATA_DIR=/srv/app/data
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
# Install necessary packages to run CKAN
RUN apt-get update && \
apt-get install --no-install-recommends -y \
gettext \
curl \
ca-certificates \
libpq5 \
git \
postgresql-client \
python3 \
python3-distutils \
libpython3.8 \
libmagic1 \
libpcre3 \
libxslt1.1 \
libxml2 \
tzdata \
apache2-utils && \
rm -rf /var/lib/apt/lists/* && \
# Create SRC_DIR
mkdir -p ${SRC_DIR} && \
# Link python to python3
ln -s /usr/bin/python3 /usr/bin/python
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
python ${SRC_DIR}/get-pip.py
# Downgrade setuptools so that CKAN requirements can be built
RUN pip install setuptools==44.1.0
# Get artifacts from build stages
COPY --from=ckanbuild /wheels /srv/app/wheels
COPY --from=extbuild /wheels /srv/app/ext_wheels
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
# Additional install steps for build stages artifacts
RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0
# Create a local user and group to run the app
RUN groupadd -g 92 ckan && \
useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan
WORKDIR ${CKAN_DIR}
# Install CKAN
RUN pip install -e /srv/app/src/ckan && \
cp who.ini ${APP_DIR} && \
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
# Install default CKAN extensions
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
# Create and update CKAN config
# Generate CKAN config
ckan generate config ${APP_DIR}/production.ini && \
# Configure plugins
ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Create the data directory
mkdir ${DATA_DIR} && \
# Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created
ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \
# Set the default level for extensions to INFO
ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
# Change ownership to app user
chown -R ckan:ckan /srv/app
# Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
# Copy necessary scripts
COPY setup/app ${APP_DIR}
WORKDIR ${APP_DIR}
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir docker-entrypoint.d
# Create afterinit directory for children image scripts
ONBUILD RUN mkdir docker-afterinit.d
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
USER ckan
CMD ["/srv/app/start_ckan.sh"]

0
images/ckan/2.9/patches/.gitkeep
View File

11
images/ckan/2.9/patches/00_patch_sql_url.patch
View File

@ -1,11 +0,0 @@
--- ckan/ckan/model/__init__.py
+++ ckan/ckan/model/__init__.py
@@ -276,7 +276,7 @@ class Repository():
self.reset_alembic_output()
alembic_config = AlembicConfig(self._alembic_ini)
alembic_config.set_main_option(
- "sqlalchemy.url", config.get("sqlalchemy.url")
+ "sqlalchemy.url", config.get("sqlalchemy.url").replace('%', '%%')
)
try:
sqlalchemy_migrate_version = self.metadata.bind.execute(

11
images/ckan/2.9/patches/01_patch_postgres_username_split.patch
View File

@ -1,11 +0,0 @@
--- ckan/ckanext/datastore/backend/postgres.py
+++ ckan/ckanext/datastore/backend/postgres.py
@@ -1809,7 +1809,7 @@ class DatastorePostgresqlBackend(DatastoreBackend):
read only user.
'''
write_connection = self._get_write_engine().connect()
- read_connection_user = sa_url.make_url(self.read_url).username
+ read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0]
drop_foo_sql = u'DROP TABLE IF EXISTS _foo'

5
images/ckan/2.9/scripts/apply_ckan_patches.sh
View File

@ -1,5 +0,0 @@
#!/bin/bash
shopt -s nullglob
for patch in patches/*.patch; do
/usr/bin/patch -p0 -i $patch
done

39
images/ckan/2.9/setup/app/basic-auth-uwsgi.conf
View File

@ -1,39 +0,0 @@
[uwsgi]
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
socket = /tmp/uwsgi.sock
uid = ckan
gid = ckan
http = :5000
master = true
enable-threads = true
lazy-apps = true
gevent-early-monkey-patch = true
vacuum = true
single-interpreter= true
die-on-term = true
need-app = true
auto-procname = true
wsgi-file = /srv/app/wsgi.py
module = wsgi:application
gevent = 2000
callable = application
paste = config:/srv/app/production.ini
paste-logger = /srv/app/production.ini
post-buffering = 1
buffer-size= 12288
max-requests = 3000
max-worker-lifetime = 3600
reload-on-rss = 4096
worker-reload-mercy = 60
socket-timeout = 300
queue = 1000
queue-blocksize = 204800
static-gzip-all = true
listen = 1000
http-timeout = 1000
http-headers-timeout = 1000
http-connect-timeout = 1000

4
images/ckan/2.9/setup/app/extra_scripts.sh
View File

@ -1,4 +0,0 @@
#!/bin/bash
# this is called before uwsgi is executed
# uset his to add extra scripts before ckan is started

25
images/ckan/2.9/setup/app/maintenance/api/3/action/status_show/index.html
View File

@ -1,25 +0,0 @@
<!--
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<title>Maintenance</title>
</head>
<body>
<h1>Maintenance</h1>
<p>Our data portal is currently in maintenance, please try in a while.</p>
</body>
</html>

25
images/ckan/2.9/setup/app/maintenance/index.html
View File

@ -1,25 +0,0 @@
<!--
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<title>Maintenance</title>
</head>
<body>
<h1>Maintenance</h1>
<p>Our data portal is currently in maintenance, please try in a while.</p>
</body>
</html>

35
images/ckan/2.9/setup/app/maintenance/serve.py
View File

@ -1,35 +0,0 @@
"""
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
import os
PORT = 5000
web_dir = os.path.join(os.path.dirname(__file__))
os.chdir(web_dir)
def run(server_class=ThreadingHTTPServer, handler_class=SimpleHTTPRequestHandler):
server_address = ("0.0.0.0", PORT)
httpd = server_class(server_address, handler_class)
print("Starting maintenance mode")
httpd.serve_forever()
if __name__ == "__main__":
run()

228
images/ckan/2.9/setup/app/prerun.py
View File

@ -1,228 +0,0 @@
"""
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from multiprocessing import connection
import os
import sys
import subprocess
import psycopg2
from sqlalchemy.engine.url import make_url
import urllib.request, urllib.error, urllib.parse, base64
import re
import json
import time
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
RETRY = 5
def check_db_connection(retry=None):
print('[prerun] Start check_db_connection...')
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
try:
db_user = make_url(conn_str).username
db_passwd = make_url(conn_str).password
db_host = make_url(conn_str).host
db_name = make_url(conn_str).database
connection = psycopg2.connect(user=db_user,
host=db_host,
password=db_passwd,
database=db_name)
except psycopg2.Error as e:
print((str(e)))
print('[prerun] Unable to connect to the database...try again in a while.')
import time
time.sleep(10)
check_db_connection(retry = retry - 1)
else:
connection.close()
def check_solr_connection(retry=None):
print('[prerun] Start check_solr_connection...')
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
url = os.environ.get('CKAN_SOLR_URL', '')
username = os.environ.get('CKAN_SOLR_USER', '')
password = os.environ.get('CKAN_SOLR_PASSWORD', '')
search_url = '{url}/schema/name?wt=json'.format(url=url)
try:
if not username:
connection = urllib.request.urlopen(search_url)
else:
request = urllib.request.Request(search_url)
base64string = base64.b64encode(bytes('%s:%s' % (username, password),'ascii'))
request.add_header("Authorization", "Basic %s" % base64string.decode('utf-8'))
connection = urllib.request.urlopen(request)
except urllib.error.URLError as e:
print('[prerun] Unable to connect to solr...try again in a while.')
import time
time.sleep(10)
check_solr_connection(retry = retry - 1)
else:
import re
conn_info = connection.read()
schema_name = json.loads(conn_info)
if 'ckan' in schema_name['name']:
print('[prerun] Succesfully connected to solr and CKAN schema loaded')
else:
print('[prerun] Succesfully connected to solr, but CKAN schema not found')
sys.exit(1)
def init_db():
print('[prerun] Start init_db...')
db_command = ['ckan', '-c', ckan_ini, 'db', 'init']
print('[prerun] Initializing or upgrading db - start using ckan db init')
try:
# run init scripts
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
print('[prerun] Initializing or upgrading db - end')
except subprocess.CalledProcessError as e:
if 'OperationalError' in str(e.output):
print(e.output.decode('utf-8'))
print('[prerun] Database not ready, waiting a bit before exit...')
import time
time.sleep(5)
sys.exit(1)
else:
print(e.output.decode('utf-8'))
raise e
print('[prerun] Initializing or upgrading db - finish')
def init_datastore():
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print('[prerun] Skipping datastore initialization')
return
datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore',
'set-permissions']
db_user = make_url(conn_str).username
db_passwd = make_url(conn_str).password
db_host = make_url(conn_str).host
db_name = make_url(conn_str).database
connection = psycopg2.connect(user=db_user,
host=db_host,
password=db_passwd,
database=db_name)
cursor = connection.cursor()
print('[prerun] Initializing datastore db - start')
try:
datastore_perms = subprocess.Popen(
datastore_perms_command,
stdout=subprocess.PIPE)
perms_sql = datastore_perms.stdout.read()
perms_sql = perms_sql.decode('utf-8')
perms_sql = perms_sql.replace("@"+db_host, "")
# Remove internal pg command as psycopg2 does not like it
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
cursor.execute(perms_sql)
for notice in connection.notices:
print(notice)
connection.commit()
print('[prerun] Initializing datastore db - end')
print((datastore_perms.stdout.read()))
except psycopg2.Error as e:
print('[prerun] Could not initialize datastore')
print(e.decode('utf-8'))
except subprocess.CalledProcessError as e:
if 'OperationalError' in str(e.output):
print(e.output.decode('utf-8'))
print('[prerun] Database not ready, waiting a bit before exit...')
time.sleep(5)
sys.exit(1)
else:
print(e.output.decode('utf-8'))
raise e
finally:
cursor.close()
connection.close()
def create_sysadmin():
print('[prerun] Start create_sysadmin...')
name = os.environ.get('CKAN_SYSADMIN_NAME')
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
if name and password and email:
# Check if user exists
command = ['ckan', '-c', ckan_ini, 'user', 'show', name]
out = subprocess.check_output(command)
if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')):
print('[prerun] Sysadmin user exists, skipping creation')
return
# Create user
command = ['ckan', '-c', ckan_ini, 'user', 'add',
name,
'password=' + password,
'email=' + email]
subprocess.call(command)
print(('[prerun] Created user {0}'.format(name)))
# Make it sysadmin
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
name]
subprocess.call(command)
print(('[prerun] Made user {0} a sysadmin'.format(name)))
if __name__ == '__main__':
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
if maintenance:
print('[prerun] Maintenance mode, skipping setup...')
else:
check_db_connection()
check_solr_connection()
init_db()
if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
init_datastore()
create_sysadmin()

76
images/ckan/2.9/setup/app/start_ckan.sh
View File

@ -1,76 +0,0 @@
#!/bin/bash
# Run any startup scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-entrypoint.d" ]]
then
for f in ${APP_DIR}/docker-entrypoint.d/*; do
case "$f" in
*.sh) echo "$0: Running init file $f"; . "$f" ;;
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Add session secret from chart
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET || -z $JWT_ENCODE_SECRET || -v $JWT_ENCODE_SECRET || -z $JWT_DECODE_SECRET || -v $JWT_DECODE_SECRET ]];then
echo "Not all environment variables are set. Generating sessions..."
else
echo "Setting session secrets from environment variables"
ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_ENCODE_SECRET"
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_DECODE_SECRET"
fi
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
then
echo "Setting secrets in ini file"
ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
ckan config-tool $APP_DIR/production.ini "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_SECRET"
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_SECRET"
fi
# Run the prerun script to init CKAN and create the default admin user
python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }
# Check if we are in maintenance mode and if yes serve the maintenance pages
if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi
# Run any after prerun/init scripts provided by images extending this one
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
then
for f in ${APP_DIR}/docker-afterinit.d/*; do
case "$f" in
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then
if [ "$PASSWORD_PROTECT" = true ]
then
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
then
# Generate htpasswd file for basicauth
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
# Start uwsgi with basicauth
uwsgi --ini /srv/app/basic-auth-uwsgi.conf -p ${UWSGI_PROC_NO:-2} --pcre-jit
else
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
exit 1
fi
else
# Start uwsgi
echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
uwsgi --ini /srv/app/uwsgi.conf -p ${UWSGI_PROC_NO:-2}
fi
else
echo "[prerun] failed...not starting CKAN."
fi

37
images/ckan/2.9/setup/app/uwsgi.conf
View File

@ -1,37 +0,0 @@
[uwsgi]
socket = /tmp/uwsgi.sock
uid = ckan
gid = ckan
http = :5000
master = true
enable-threads = true
lazy-apps = true
gevent-early-monkey-patch = true
vacuum = true
single-interpreter= true
die-on-term = true
need-app = true
auto-procname = true
wsgi-file = /srv/app/wsgi.py
module = wsgi:application
gevent = 2000
callable = application
paste = config:/srv/app/production.ini
paste-logger = /srv/app/production.ini
post-buffering = 1
buffer-size= 12288
max-requests = 3000
max-worker-lifetime = 3600
reload-on-rss = 4096
worker-reload-mercy = 60
socket-timeout = 300
queue = 1000
queue-blocksize = 204800
static-gzip-all = true
listen = 1000
http-timeout = 1000
http-headers-timeout = 1000
http-connect-timeout = 1000

28
images/ckan/2.9/setup/app/wsgi.py
View File

@ -1,28 +0,0 @@
"""
Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
# -*- coding: utf-8 -*-
import os
from ckan.config.middleware import make_app
from ckan.cli import CKANConfigLoader
from logging.config import fileConfig as loggingFileConfig
config_filepath = os.path.join(
os.path.dirname(os.path.abspath(__file__)), u'production.ini')
abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
loggingFileConfig(config_filepath)
config = CKANConfigLoader(config_filepath).get_config()
application = make_app(config)