deleted unused files
This commit is contained in:
Elia Bellavista
parent
a9989167f9
commit
d7ee7d66f3
|
|
@ -1,36 +0,0 @@
|
||||||
# Runtime configuration of CKAN enabled through ckanext-envvars
|
|
||||||
# Information about how it works: https://github.com/okfn/ckanext-envvars
|
|
||||||
# Note that variables here take presedence over build/up time variables in .env
|
|
||||||
|
|
||||||
# Set to true to disable CKAN from starting and serve a maintenance page
|
|
||||||
MAINTENANCE_MODE=false
|
|
||||||
|
|
||||||
# General Settings
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_MAX_UPLOAD_SIZE_MB=10
|
|
||||||
CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
|
|
||||||
# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
|
|
||||||
CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
|
|
||||||
CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
|
|
||||||
# CKAN Plugins
|
|
||||||
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
|
|
||||||
# CKAN requires storage path to be set in order for filestore to be enabled
|
|
||||||
CKAN__STORAGE_PATH=/srv/app/data
|
|
||||||
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
|
|
||||||
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
|
|
||||||
CKAN_SYSADMIN_NAME=sysadmin
|
|
||||||
CKAN_SYSADMIN_PASSWORD=password
|
|
||||||
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
|
|
||||||
|
|
||||||
# Email settings
|
|
||||||
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
|
|
||||||
CKAN_SMTP_STARTTLS=True
|
|
||||||
CKAN_SMTP_USER=user
|
|
||||||
CKAN_SMTP_PASSWORD=pass
|
|
||||||
CKAN_SMTP_MAIL_FROM=ckan@localhost
|
|
||||||
|
|
||||||
# Datapusher configuration
|
|
||||||
CKAN__DATAPUSHER__URL=http://datapusher:8000
|
|
||||||
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
|
|
||||||
|
|
@ -1,32 +0,0 @@
|
||||||
# Variables in this file will be used as build arguments when running
|
|
||||||
# docker-compose build and docker-compose up
|
|
||||||
# Verify correct substitution with "docker-compose config"
|
|
||||||
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
|
|
||||||
# docker-compose down -v
|
|
||||||
# docker-compose build
|
|
||||||
# docker-compose up -d
|
|
||||||
|
|
||||||
# Database
|
|
||||||
POSTGRES_PASSWORD=ckan
|
|
||||||
POSTGRES_PORT=5432
|
|
||||||
DATASTORE_READONLY_PASSWORD=datastore
|
|
||||||
|
|
||||||
# CKAN
|
|
||||||
CKAN_VERSION=2.8.12
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_MAX_UPLOAD_SIZE_MB=10
|
|
||||||
|
|
||||||
# Datapusher
|
|
||||||
DATAPUSHER_VERSION=0.0.17
|
|
||||||
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
|
|
||||||
DATAPUSHER_CHUNK_SIZE=16384
|
|
||||||
DATAPUSHER_CHUNK_INSERT_ROWS=250
|
|
||||||
DATAPUSHER_DOWNLOAD_TIMEOUT=30
|
|
||||||
DATAPUSHER_SSL_VERIFY=False
|
|
||||||
DATAPUSHER_REWRITE_RESOURCES=True
|
|
||||||
DATAPUSHER_REWRITE_URL=http://ckan:5000
|
|
||||||
|
|
||||||
# Redis
|
|
||||||
REDIS_VERSION=6.0.7
|
|
||||||
|
|
@ -1,93 +0,0 @@
|
||||||
# docker-compose build && docker-compose up -d
|
|
||||||
version: "3"
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
ckan_data:
|
|
||||||
pg_data:
|
|
||||||
solr_data:
|
|
||||||
|
|
||||||
services:
|
|
||||||
ckan:
|
|
||||||
container_name: ckan
|
|
||||||
image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION}
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- db
|
|
||||||
- solr
|
|
||||||
ports:
|
|
||||||
- "0.0.0.0:${CKAN_PORT}:5000"
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
|
||||||
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
|
||||||
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
|
||||||
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
- CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
|
||||||
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
volumes:
|
|
||||||
- ckan_data:/srv/app/data
|
|
||||||
|
|
||||||
datapusher:
|
|
||||||
container_name: datapusher
|
|
||||||
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
ports:
|
|
||||||
- "8000:8000"
|
|
||||||
environment:
|
|
||||||
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
|
|
||||||
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
|
|
||||||
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
|
|
||||||
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
|
|
||||||
- DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
|
|
||||||
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
|
|
||||||
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
|
|
||||||
|
|
||||||
|
|
||||||
db:
|
|
||||||
container_name: db
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: postgresql/Dockerfile
|
|
||||||
args:
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
environment:
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- PGDATA=/var/lib/postgresql/data/db
|
|
||||||
volumes:
|
|
||||||
- pg_data:/var/lib/postgresql/data
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "pg_isready", "-U", "ckan"]
|
|
||||||
|
|
||||||
solr:
|
|
||||||
container_name: solr
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: solr/Dockerfile
|
|
||||||
args:
|
|
||||||
- CKAN_VERSION=${CKAN_VERSION}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
volumes:
|
|
||||||
- solr_data:/opt/solr/server/solr/ckan/data
|
|
||||||
|
|
||||||
redis:
|
|
||||||
container_name: redis
|
|
||||||
image: redis:${REDIS_VERSION}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
|
|
||||||
networks:
|
|
||||||
frontend:
|
|
||||||
backend:
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
FROM postgis/postgis:11-3.3-alpine
|
|
||||||
|
|
||||||
# Allow connections; we don't map out any ports so only linked docker containers can connect
|
|
||||||
RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf
|
|
||||||
|
|
||||||
# Customize default user/pass/db
|
|
||||||
ENV POSTGRES_DB ckan
|
|
||||||
ENV POSTGRES_USER ckan
|
|
||||||
ARG POSTGRES_PASSWORD
|
|
||||||
ARG DS_RO_PASS
|
|
||||||
|
|
||||||
# Include datastore setup scripts
|
|
||||||
COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
|
|
||||||
CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
|
|
||||||
CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
|
|
||||||
GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;
|
|
||||||
EOSQL
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
ALTER VIEW geometry_columns OWNER TO ckan;
|
|
||||||
ALTER TABLE spatial_ref_sys OWNER TO ckan;
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
FROM solr:6.6.6
|
|
||||||
|
|
||||||
# Enviroment
|
|
||||||
ENV SOLR_CORE ckan
|
|
||||||
ENV SOLR_VERSION 6.6.6
|
|
||||||
|
|
||||||
# Build Arguments
|
|
||||||
ARG CKAN_VERSION
|
|
||||||
|
|
||||||
# Create Directories
|
|
||||||
RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf
|
|
||||||
RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data
|
|
||||||
|
|
||||||
# Adding Files
|
|
||||||
COPY ./solr/solrconfig-$CKAN_VERSION.xml /opt/solr/server/solr/$SOLR_CORE/conf/
|
|
||||||
ADD https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.xml \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
|
|
||||||
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
|
|
||||||
/opt/solr/server/solr/$SOLR_CORE/conf/
|
|
||||||
|
|
||||||
# Create Core.properties
|
|
||||||
RUN mv /opt/solr/server/solr/$SOLR_CORE/conf/solrconfig-$CKAN_VERSION.xml /opt/solr/server/solr/$SOLR_CORE/conf/solrconfig.xml && \
|
|
||||||
echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties
|
|
||||||
|
|
||||||
# Giving ownership to Solr
|
|
||||||
|
|
||||||
USER root
|
|
||||||
|
|
||||||
RUN chown -R "$SOLR_USER:$SOLR_USER" /opt/solr/server/solr/$SOLR_CORE
|
|
||||||
|
|
||||||
# User
|
|
||||||
USER $SOLR_USER:$SOLR_USER
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
solrconfig-2.8.10.xml
|
|
||||||
|
|
@ -1,343 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
<!--solrconfig.xml documentation [https://wiki.apache.org/solr/SolrConfigXml]-->
|
|
||||||
<config>
|
|
||||||
|
|
||||||
<luceneMatchVersion>6.0.0</luceneMatchVersion>
|
|
||||||
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
|
|
||||||
|
|
||||||
<dataDir>${solr.data.dir:}</dataDir>
|
|
||||||
|
|
||||||
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}" />
|
|
||||||
|
|
||||||
<codecFactory class="solr.SchemaCodecFactory" />
|
|
||||||
|
|
||||||
<indexConfig>
|
|
||||||
<lockType>${solr.lock.type:native}</lockType>
|
|
||||||
</indexConfig>
|
|
||||||
|
|
||||||
<jmx />
|
|
||||||
|
|
||||||
<updateHandler class="solr.DirectUpdateHandler2">
|
|
||||||
|
|
||||||
<updateLog>
|
|
||||||
<str name="dir">${solr.ulog.dir:}</str>
|
|
||||||
<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
|
|
||||||
</updateLog>
|
|
||||||
|
|
||||||
<autoCommit>
|
|
||||||
<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
|
|
||||||
<openSearcher>false</openSearcher>
|
|
||||||
</autoCommit>
|
|
||||||
|
|
||||||
<autoSoftCommit>
|
|
||||||
<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
|
|
||||||
</autoSoftCommit>
|
|
||||||
|
|
||||||
</updateHandler>
|
|
||||||
|
|
||||||
<query>
|
|
||||||
|
|
||||||
<maxBooleanClauses>1024</maxBooleanClauses>
|
|
||||||
<filterCache class="solr.FastLRUCache" size="512" initialSize="512" autowarmCount="0" />
|
|
||||||
<queryResultCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
|
|
||||||
<documentCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
|
|
||||||
<cache name="perSegFilter" class="solr.search.LRUCache" size="10" initialSize="0" autowarmCount="10" regenerator="solr.NoOpRegenerator" />
|
|
||||||
<enableLazyFieldLoading>true</enableLazyFieldLoading>
|
|
||||||
<queryResultWindowSize>20</queryResultWindowSize>
|
|
||||||
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
|
|
||||||
|
|
||||||
<listener event="newSearcher" class="solr.QuerySenderListener">
|
|
||||||
<arr name="queries" />
|
|
||||||
</listener>
|
|
||||||
|
|
||||||
<listener event="firstSearcher" class="solr.QuerySenderListener">
|
|
||||||
<arr name="queries" />
|
|
||||||
</listener>
|
|
||||||
|
|
||||||
<useColdSearcher>false</useColdSearcher>
|
|
||||||
<maxWarmingSearchers>2</maxWarmingSearchers>
|
|
||||||
|
|
||||||
</query>
|
|
||||||
|
|
||||||
<requestDispatcher handleSelect="false">
|
|
||||||
|
|
||||||
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048" addHttpRequestToContext="false" />
|
|
||||||
<httpCaching never304="true" />
|
|
||||||
|
|
||||||
</requestDispatcher>
|
|
||||||
|
|
||||||
<requestHandler name="/select" class="solr.SearchHandler">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
<int name="rows">10</int>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<requestHandler name="/query" class="solr.SearchHandler">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
<str name="wt">json</str>
|
|
||||||
<str name="indent">true</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<requestHandler name="/browse" class="solr.SearchHandler" useParams="query,facets,velocity,browse">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="df">_text_</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</initParams>
|
|
||||||
|
|
||||||
<initParams path="/update/**">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="update.chain">add-unknown-fields-to-the-schema</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</initParams>
|
|
||||||
|
|
||||||
<requestHandler name="/update/extract" startup="lazy" class="solr.extraction.ExtractingRequestHandler">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="lowernames">true</str>
|
|
||||||
<str name="fmap.meta">ignored_</str>
|
|
||||||
<str name="fmap.content">_text_</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
|
|
||||||
<requestHandler name="/analysis/document" class="solr.DocumentAnalysisRequestHandler" startup="lazy" />
|
|
||||||
|
|
||||||
<requestHandler name="/debug/dump" class="solr.DumpRequestHandler">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
<str name="echoHandler">true</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent name="spellcheck" class="solr.SpellCheckComponent">
|
|
||||||
|
|
||||||
<str name="queryAnalyzerFieldType">text_general</str>
|
|
||||||
|
|
||||||
<lst name="spellchecker">
|
|
||||||
<str name="name">default</str>
|
|
||||||
<str name="field">_text_</str>
|
|
||||||
<str name="classname">solr.DirectSolrSpellChecker</str>
|
|
||||||
<str name="distanceMeasure">internal</str>
|
|
||||||
<float name="accuracy">0.5</float>
|
|
||||||
<int name="maxEdits">2</int>
|
|
||||||
<int name="minPrefix">1</int>
|
|
||||||
<int name="maxInspections">5</int>
|
|
||||||
<int name="minQueryLength">4</int>
|
|
||||||
<float name="maxQueryFrequency">0.01</float>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</searchComponent>
|
|
||||||
|
|
||||||
<requestHandler name="/spell" class="solr.SearchHandler" startup="lazy">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="spellcheck.dictionary">default</str>
|
|
||||||
<str name="spellcheck">on</str>
|
|
||||||
<str name="spellcheck.extendedResults">true</str>
|
|
||||||
<str name="spellcheck.count">10</str>
|
|
||||||
<str name="spellcheck.alternativeTermCount">5</str>
|
|
||||||
<str name="spellcheck.maxResultsForSuggest">5</str>
|
|
||||||
<str name="spellcheck.collate">true</str>
|
|
||||||
<str name="spellcheck.collateExtendedResults">true</str>
|
|
||||||
<str name="spellcheck.maxCollationTries">10</str>
|
|
||||||
<str name="spellcheck.maxCollations">5</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
<arr name="last-components">
|
|
||||||
<str>spellcheck</str>
|
|
||||||
</arr>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent name="tvComponent" class="solr.TermVectorComponent" />
|
|
||||||
|
|
||||||
<requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<bool name="tv">true</bool>
|
|
||||||
</lst>
|
|
||||||
<arr name="last-components">
|
|
||||||
<str>tvComponent</str>
|
|
||||||
</arr>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent name="terms" class="solr.TermsComponent" />
|
|
||||||
|
|
||||||
<requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<bool name="terms">true</bool>
|
|
||||||
<bool name="distrib">false</bool>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
<arr name="components">
|
|
||||||
<str>terms</str>
|
|
||||||
</arr>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent name="elevator" class="solr.QueryElevationComponent">
|
|
||||||
|
|
||||||
<str name="queryFieldType">string</str>
|
|
||||||
<str name="config-file">elevate.xml</str>
|
|
||||||
|
|
||||||
</searchComponent>
|
|
||||||
|
|
||||||
<requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
</lst>
|
|
||||||
<arr name="last-components">
|
|
||||||
<str>elevator</str>
|
|
||||||
</arr>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent class="solr.HighlightComponent" name="highlight">
|
|
||||||
|
|
||||||
<highlighting>
|
|
||||||
|
|
||||||
<fragmenter name="gap" default="true" class="solr.highlight.GapFragmenter">
|
|
||||||
<lst name="defaults">
|
|
||||||
<int name="hl.fragsize">100</int>
|
|
||||||
</lst>
|
|
||||||
</fragmenter>
|
|
||||||
|
|
||||||
<fragmenter name="regex" class="solr.highlight.RegexFragmenter">
|
|
||||||
<lst name="defaults">
|
|
||||||
<int name="hl.fragsize">70</int>
|
|
||||||
<float name="hl.regex.slop">0.5</float>
|
|
||||||
<str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str>
|
|
||||||
</lst>
|
|
||||||
</fragmenter>
|
|
||||||
|
|
||||||
<formatter name="html" default="true" class="solr.highlight.HtmlFormatter">
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="hl.simple.pre"><![CDATA[<em>]]></str>
|
|
||||||
<str name="hl.simple.post"><![CDATA[</em>]]></str>
|
|
||||||
</lst>
|
|
||||||
</formatter>
|
|
||||||
|
|
||||||
<encoder name="html" class="solr.highlight.HtmlEncoder" />
|
|
||||||
<fragListBuilder name="simple" class="solr.highlight.SimpleFragListBuilder" />
|
|
||||||
<fragListBuilder name="single" class="solr.highlight.SingleFragListBuilder" />
|
|
||||||
<fragListBuilder name="weighted" default="true" class="solr.highlight.WeightedFragListBuilder" />
|
|
||||||
<fragmentsBuilder name="default" default="true" class="solr.highlight.ScoreOrderFragmentsBuilder" />
|
|
||||||
<fragmentsBuilder name="colored" class="solr.highlight.ScoreOrderFragmentsBuilder">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="hl.tag.pre"><![CDATA[<b style="background:yellow">,<b style="background:lawgreen">,
|
|
||||||
<b style="background:aquamarine">,<b style="background:magenta">,
|
|
||||||
<b style="background:palegreen">,<b style="background:coral">,
|
|
||||||
<b style="background:wheat">,<b style="background:khaki">,
|
|
||||||
<b style="background:lime">,<b style="background:deepskyblue">]]></str>
|
|
||||||
<str name="hl.tag.post"><![CDATA[</b>]]></str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</fragmentsBuilder>
|
|
||||||
|
|
||||||
<boundaryScanner name="default" default="true" class="solr.highlight.SimpleBoundaryScanner">
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="hl.bs.maxScan">10</str>
|
|
||||||
<str name="hl.bs.chars">.,!?</str>
|
|
||||||
</lst>
|
|
||||||
</boundaryScanner>
|
|
||||||
|
|
||||||
<boundaryScanner name="breakIterator" class="solr.highlight.BreakIteratorBoundaryScanner">
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="hl.bs.type">WORD</str>
|
|
||||||
<str name="hl.bs.language">en</str>
|
|
||||||
<str name="hl.bs.country">US</str>
|
|
||||||
</lst>
|
|
||||||
</boundaryScanner>
|
|
||||||
|
|
||||||
</highlighting>
|
|
||||||
|
|
||||||
</searchComponent>
|
|
||||||
|
|
||||||
<schemaFactory class="ClassicIndexSchemaFactory" />
|
|
||||||
|
|
||||||
<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
|
|
||||||
<processor class="solr.UUIDUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.LogUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.DistributedUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.RemoveBlankFieldUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.FieldNameMutatingUpdateProcessorFactory">
|
|
||||||
<str name="pattern">[^\w-\.]</str>
|
|
||||||
<str name="replacement">_</str>
|
|
||||||
</processor>
|
|
||||||
<processor class="solr.ParseBooleanFieldUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.ParseLongFieldUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
|
|
||||||
<arr name="format">
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ssZ</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mmZ</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss.SSS</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss,SSS</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ssZ</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss</str>
|
|
||||||
<str>yyyy-MM-dd HH:mmZ</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm</str>
|
|
||||||
<str>yyyy-MM-dd</str>
|
|
||||||
</arr>
|
|
||||||
</processor>
|
|
||||||
<processor class="solr.RunUpdateProcessorFactory" />
|
|
||||||
</updateRequestProcessorChain>
|
|
||||||
|
|
||||||
<queryResponseWriter name="json" class="solr.JSONResponseWriter">
|
|
||||||
<str name="content-type">text/plain; charset=UTF-8</str>
|
|
||||||
</queryResponseWriter>
|
|
||||||
|
|
||||||
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter" startup="lazy">
|
|
||||||
<str name="template.base.dir">${velocity.template.base.dir:}</str>
|
|
||||||
<str name="solr.resource.loader.enabled">${velocity.solr.resource.loader.enabled:true}</str>
|
|
||||||
<str name="params.resource.loader.enabled">${velocity.params.resource.loader.enabled:false}</str>
|
|
||||||
</queryResponseWriter>
|
|
||||||
|
|
||||||
<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
|
|
||||||
<int name="xsltCacheLifetimeSeconds">5</int>
|
|
||||||
</queryResponseWriter>
|
|
||||||
|
|
||||||
</config>
|
|
||||||
|
|
@ -1,345 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
<!--solrconfig.xml documentation [https://wiki.apache.org/solr/SolrConfigXml]-->
|
|
||||||
<config>
|
|
||||||
|
|
||||||
<luceneMatchVersion>6.0.0</luceneMatchVersion>
|
|
||||||
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
|
|
||||||
|
|
||||||
<dataDir>${solr.data.dir:}</dataDir>
|
|
||||||
|
|
||||||
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}" />
|
|
||||||
|
|
||||||
<codecFactory class="solr.SchemaCodecFactory" />
|
|
||||||
|
|
||||||
<indexConfig>
|
|
||||||
<lockType>${solr.lock.type:native}</lockType>
|
|
||||||
</indexConfig>
|
|
||||||
|
|
||||||
<jmx />
|
|
||||||
|
|
||||||
<updateHandler class="solr.DirectUpdateHandler2">
|
|
||||||
|
|
||||||
<updateLog>
|
|
||||||
<str name="dir">${solr.ulog.dir:}</str>
|
|
||||||
<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
|
|
||||||
</updateLog>
|
|
||||||
|
|
||||||
<autoCommit>
|
|
||||||
<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
|
|
||||||
<openSearcher>false</openSearcher>
|
|
||||||
</autoCommit>
|
|
||||||
|
|
||||||
<autoSoftCommit>
|
|
||||||
<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
|
|
||||||
</autoSoftCommit>
|
|
||||||
|
|
||||||
</updateHandler>
|
|
||||||
|
|
||||||
<query>
|
|
||||||
|
|
||||||
<maxBooleanClauses>1024</maxBooleanClauses>
|
|
||||||
<filterCache class="solr.FastLRUCache" size="512" initialSize="512" autowarmCount="0" />
|
|
||||||
<queryResultCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
|
|
||||||
<documentCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
|
|
||||||
<cache name="perSegFilter" class="solr.search.LRUCache" size="10" initialSize="0" autowarmCount="10" regenerator="solr.NoOpRegenerator" />
|
|
||||||
<enableLazyFieldLoading>true</enableLazyFieldLoading>
|
|
||||||
<queryResultWindowSize>20</queryResultWindowSize>
|
|
||||||
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
|
|
||||||
|
|
||||||
<listener event="newSearcher" class="solr.QuerySenderListener">
|
|
||||||
<arr name="queries" />
|
|
||||||
</listener>
|
|
||||||
|
|
||||||
<listener event="firstSearcher" class="solr.QuerySenderListener">
|
|
||||||
<arr name="queries" />
|
|
||||||
</listener>
|
|
||||||
|
|
||||||
<useColdSearcher>false</useColdSearcher>
|
|
||||||
<maxWarmingSearchers>2</maxWarmingSearchers>
|
|
||||||
|
|
||||||
</query>
|
|
||||||
|
|
||||||
<requestDispatcher handleSelect="false">
|
|
||||||
|
|
||||||
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048" addHttpRequestToContext="false" />
|
|
||||||
<httpCaching never304="true" />
|
|
||||||
|
|
||||||
</requestDispatcher>
|
|
||||||
|
|
||||||
<requestHandler name="/select" class="solr.SearchHandler">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
<int name="rows">10</int>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<requestHandler name="/query" class="solr.SearchHandler">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
<str name="wt">json</str>
|
|
||||||
<str name="indent">true</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<requestHandler name="/browse" class="solr.SearchHandler" useParams="query,facets,velocity,browse">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="df">_text_</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</initParams>
|
|
||||||
|
|
||||||
<initParams path="/update/**">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="update.chain">add-unknown-fields-to-the-schema</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</initParams>
|
|
||||||
|
|
||||||
<requestHandler name="/update/extract" startup="lazy" class="solr.extraction.ExtractingRequestHandler">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="lowernames">true</str>
|
|
||||||
<str name="fmap.meta">ignored_</str>
|
|
||||||
<str name="fmap.content">_text_</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
|
|
||||||
<requestHandler name="/analysis/document" class="solr.DocumentAnalysisRequestHandler" startup="lazy" />
|
|
||||||
|
|
||||||
<requestHandler name="/debug/dump" class="solr.DumpRequestHandler">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
<str name="echoHandler">true</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent name="spellcheck" class="solr.SpellCheckComponent">
|
|
||||||
|
|
||||||
<str name="queryAnalyzerFieldType">text_general</str>
|
|
||||||
|
|
||||||
<lst name="spellchecker">
|
|
||||||
<str name="name">default</str>
|
|
||||||
<str name="field">_text_</str>
|
|
||||||
<str name="classname">solr.DirectSolrSpellChecker</str>
|
|
||||||
<str name="distanceMeasure">internal</str>
|
|
||||||
<float name="accuracy">0.5</float>
|
|
||||||
<int name="maxEdits">2</int>
|
|
||||||
<int name="minPrefix">1</int>
|
|
||||||
<int name="maxInspections">5</int>
|
|
||||||
<int name="minQueryLength">4</int>
|
|
||||||
<float name="maxQueryFrequency">0.01</float>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</searchComponent>
|
|
||||||
|
|
||||||
<requestHandler name="/spell" class="solr.SearchHandler" startup="lazy">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="spellcheck.dictionary">default</str>
|
|
||||||
<str name="spellcheck">on</str>
|
|
||||||
<str name="spellcheck.extendedResults">true</str>
|
|
||||||
<str name="spellcheck.count">10</str>
|
|
||||||
<str name="spellcheck.alternativeTermCount">5</str>
|
|
||||||
<str name="spellcheck.maxResultsForSuggest">5</str>
|
|
||||||
<str name="spellcheck.collate">true</str>
|
|
||||||
<str name="spellcheck.collateExtendedResults">true</str>
|
|
||||||
<str name="spellcheck.maxCollationTries">10</str>
|
|
||||||
<str name="spellcheck.maxCollations">5</str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
<arr name="last-components">
|
|
||||||
<str>spellcheck</str>
|
|
||||||
</arr>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent name="tvComponent" class="solr.TermVectorComponent" />
|
|
||||||
|
|
||||||
<requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<bool name="tv">true</bool>
|
|
||||||
</lst>
|
|
||||||
<arr name="last-components">
|
|
||||||
<str>tvComponent</str>
|
|
||||||
</arr>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent name="terms" class="solr.TermsComponent" />
|
|
||||||
|
|
||||||
<requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<bool name="terms">true</bool>
|
|
||||||
<bool name="distrib">false</bool>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
<arr name="components">
|
|
||||||
<str>terms</str>
|
|
||||||
</arr>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent name="elevator" class="solr.QueryElevationComponent">
|
|
||||||
|
|
||||||
<str name="queryFieldType">string</str>
|
|
||||||
<str name="config-file">elevate.xml</str>
|
|
||||||
|
|
||||||
</searchComponent>
|
|
||||||
|
|
||||||
<requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
</lst>
|
|
||||||
<arr name="last-components">
|
|
||||||
<str>elevator</str>
|
|
||||||
</arr>
|
|
||||||
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<searchComponent class="solr.HighlightComponent" name="highlight">
|
|
||||||
|
|
||||||
<highlighting>
|
|
||||||
|
|
||||||
<fragmenter name="gap" default="true" class="solr.highlight.GapFragmenter">
|
|
||||||
<lst name="defaults">
|
|
||||||
<int name="hl.fragsize">100</int>
|
|
||||||
</lst>
|
|
||||||
</fragmenter>
|
|
||||||
|
|
||||||
<fragmenter name="regex" class="solr.highlight.RegexFragmenter">
|
|
||||||
<lst name="defaults">
|
|
||||||
<int name="hl.fragsize">70</int>
|
|
||||||
<float name="hl.regex.slop">0.5</float>
|
|
||||||
<str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str>
|
|
||||||
</lst>
|
|
||||||
</fragmenter>
|
|
||||||
|
|
||||||
<formatter name="html" default="true" class="solr.highlight.HtmlFormatter">
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="hl.simple.pre"><![CDATA[<em>]]></str>
|
|
||||||
<str name="hl.simple.post"><![CDATA[</em>]]></str>
|
|
||||||
</lst>
|
|
||||||
</formatter>
|
|
||||||
|
|
||||||
<encoder name="html" class="solr.highlight.HtmlEncoder" />
|
|
||||||
<fragListBuilder name="simple" class="solr.highlight.SimpleFragListBuilder" />
|
|
||||||
<fragListBuilder name="single" class="solr.highlight.SingleFragListBuilder" />
|
|
||||||
<fragListBuilder name="weighted" default="true" class="solr.highlight.WeightedFragListBuilder" />
|
|
||||||
<fragmentsBuilder name="default" default="true" class="solr.highlight.ScoreOrderFragmentsBuilder" />
|
|
||||||
<fragmentsBuilder name="colored" class="solr.highlight.ScoreOrderFragmentsBuilder">
|
|
||||||
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="hl.tag.pre"><![CDATA[<b style="background:yellow">,<b style="background:lawgreen">,
|
|
||||||
<b style="background:aquamarine">,<b style="background:magenta">,
|
|
||||||
<b style="background:palegreen">,<b style="background:coral">,
|
|
||||||
<b style="background:wheat">,<b style="background:khaki">,
|
|
||||||
<b style="background:lime">,<b style="background:deepskyblue">]]></str>
|
|
||||||
<str name="hl.tag.post"><![CDATA[</b>]]></str>
|
|
||||||
</lst>
|
|
||||||
|
|
||||||
</fragmentsBuilder>
|
|
||||||
|
|
||||||
<boundaryScanner name="default" default="true" class="solr.highlight.SimpleBoundaryScanner">
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="hl.bs.maxScan">10</str>
|
|
||||||
<str name="hl.bs.chars">.,!?</str>
|
|
||||||
</lst>
|
|
||||||
</boundaryScanner>
|
|
||||||
|
|
||||||
<boundaryScanner name="breakIterator" class="solr.highlight.BreakIteratorBoundaryScanner">
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="hl.bs.type">WORD</str>
|
|
||||||
<str name="hl.bs.language">en</str>
|
|
||||||
<str name="hl.bs.country">US</str>
|
|
||||||
</lst>
|
|
||||||
</boundaryScanner>
|
|
||||||
|
|
||||||
</highlighting>
|
|
||||||
|
|
||||||
</searchComponent>
|
|
||||||
|
|
||||||
<schemaFactory class="ManagedIndexSchemaFactory">
|
|
||||||
<bool name="mutable">true</bool>
|
|
||||||
</schemaFactory>
|
|
||||||
|
|
||||||
<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
|
|
||||||
<processor class="solr.UUIDUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.LogUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.DistributedUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.RemoveBlankFieldUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.FieldNameMutatingUpdateProcessorFactory">
|
|
||||||
<str name="pattern">[^\w-\.]</str>
|
|
||||||
<str name="replacement">_</str>
|
|
||||||
</processor>
|
|
||||||
<processor class="solr.ParseBooleanFieldUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.ParseLongFieldUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory" />
|
|
||||||
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
|
|
||||||
<arr name="format">
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ssZ</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm:ss</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mmZ</str>
|
|
||||||
<str>yyyy-MM-dd'T'HH:mm</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss.SSS</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss,SSS</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ssZ</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm:ss</str>
|
|
||||||
<str>yyyy-MM-dd HH:mmZ</str>
|
|
||||||
<str>yyyy-MM-dd HH:mm</str>
|
|
||||||
<str>yyyy-MM-dd</str>
|
|
||||||
</arr>
|
|
||||||
</processor>
|
|
||||||
<processor class="solr.RunUpdateProcessorFactory" />
|
|
||||||
</updateRequestProcessorChain>
|
|
||||||
|
|
||||||
<queryResponseWriter name="json" class="solr.JSONResponseWriter">
|
|
||||||
<str name="content-type">text/plain; charset=UTF-8</str>
|
|
||||||
</queryResponseWriter>
|
|
||||||
|
|
||||||
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter" startup="lazy">
|
|
||||||
<str name="template.base.dir">${velocity.template.base.dir:}</str>
|
|
||||||
<str name="solr.resource.loader.enabled">${velocity.solr.resource.loader.enabled:true}</str>
|
|
||||||
<str name="params.resource.loader.enabled">${velocity.params.resource.loader.enabled:false}</str>
|
|
||||||
</queryResponseWriter>
|
|
||||||
|
|
||||||
<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
|
|
||||||
<int name="xsltCacheLifetimeSeconds">5</int>
|
|
||||||
</queryResponseWriter>
|
|
||||||
|
|
||||||
</config>
|
|
||||||
|
|
@ -1,203 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8" ?>
|
|
||||||
<!--
|
|
||||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
contributor license agreements. See the NOTICE file distributed with
|
|
||||||
this work for additional information regarding copyright ownership.
|
|
||||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
(the "License"); you may not use this file except in compliance with
|
|
||||||
the License. You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<!--
|
|
||||||
NB Please copy changes to this file into the multilingual schema:
|
|
||||||
ckanext/multilingual/solr/schema.xml
|
|
||||||
-->
|
|
||||||
|
|
||||||
<!-- We update the version when there is a backward-incompatible change to this
|
|
||||||
schema. We used to use the `version` attribute for this but this is an internal
|
|
||||||
attribute that should not be used so starting from CKAN 2.10 we use the `name`
|
|
||||||
attribute with the form `ckan-X.Y` -->
|
|
||||||
<schema name="ckan-2.10" version="1.6">
|
|
||||||
|
|
||||||
<types>
|
|
||||||
<fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
|
|
||||||
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
|
|
||||||
<fieldtype name="binary" class="solr.BinaryField"/>
|
|
||||||
<fieldType name="int" class="solr.IntPointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="float" class="solr.FloatPointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="long" class="solr.LongPointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="double" class="solr.DoublePointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="pint" class="solr.IntPointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="pfloat" class="solr.FloatPointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="plong" class="solr.LongPointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="pdouble" class="solr.DoublePointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="date" class="solr.DatePointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="pdate" class="solr.DatePointField" omitNorms="true" positionIncrementGap="0"/>
|
|
||||||
|
|
||||||
<fieldType name="pdates" class="solr.DatePointField" positionIncrementGap="0" multiValued="true"/>
|
|
||||||
<fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
|
|
||||||
<fieldType name="pints" class="solr.IntPointField" positionIncrementGap="0" multiValued="true"/>
|
|
||||||
<fieldType name="pfloats" class="solr.FloatPointField" positionIncrementGap="0" multiValued="true"/>
|
|
||||||
<fieldType name="plongs" class="solr.LongPointField" positionIncrementGap="0" multiValued="true"/>
|
|
||||||
<fieldType name="pdoubles" class="solr.DoublePointField" positionIncrementGap="0" multiValued="true"/>
|
|
||||||
|
|
||||||
<fieldType name="text" class="solr.TextField" positionIncrementGap="100">
|
|
||||||
<analyzer type="index">
|
|
||||||
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
|
||||||
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
|
|
||||||
<filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
|
|
||||||
<filter class="solr.LowerCaseFilterFactory"/>
|
|
||||||
<filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
|
|
||||||
<filter class="solr.ASCIIFoldingFilterFactory"/>
|
|
||||||
</analyzer>
|
|
||||||
<analyzer type="query">
|
|
||||||
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
|
||||||
<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
|
|
||||||
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
|
|
||||||
<filter class="solr.LowerCaseFilterFactory"/>
|
|
||||||
<filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
|
|
||||||
<filter class="solr.ASCIIFoldingFilterFactory"/>
|
|
||||||
</analyzer>
|
|
||||||
</fieldType>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- A general unstemmed text field - good if one does not know the language of the field -->
|
|
||||||
<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
|
|
||||||
<analyzer type="index">
|
|
||||||
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
|
||||||
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
|
|
||||||
<filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
|
|
||||||
<filter class="solr.LowerCaseFilterFactory"/>
|
|
||||||
</analyzer>
|
|
||||||
<analyzer type="query">
|
|
||||||
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
|
||||||
<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
|
|
||||||
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
|
|
||||||
<filter class="solr.LowerCaseFilterFactory"/>
|
|
||||||
</analyzer>
|
|
||||||
</fieldType>
|
|
||||||
|
|
||||||
<fieldType name="text_ngram" class="solr.TextField" positionIncrementGap="100">
|
|
||||||
<analyzer type="index">
|
|
||||||
<tokenizer class="solr.NGramTokenizerFactory" minGramSize="2" maxGramSize="10"/>
|
|
||||||
<filter class="solr.LowerCaseFilterFactory"/>
|
|
||||||
</analyzer>
|
|
||||||
<analyzer type="query">
|
|
||||||
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
|
||||||
<filter class="solr.LowerCaseFilterFactory"/>
|
|
||||||
</analyzer>
|
|
||||||
</fieldType>
|
|
||||||
|
|
||||||
</types>
|
|
||||||
|
|
||||||
|
|
||||||
<fields>
|
|
||||||
<field name="index_id" type="string" indexed="true" stored="true" required="true" />
|
|
||||||
<field name="id" type="string" indexed="true" stored="true" required="true" />
|
|
||||||
<field name="site_id" type="string" indexed="true" stored="true" required="true" />
|
|
||||||
<field name="title" type="text" indexed="true" stored="true" />
|
|
||||||
<field name="title_ngram" type="text_ngram" indexed="true" stored="true" />
|
|
||||||
<field name="entity_type" type="string" indexed="true" stored="true" omitNorms="true" />
|
|
||||||
<field name="dataset_type" type="string" indexed="true" stored="true" />
|
|
||||||
<field name="state" type="string" indexed="true" stored="true" omitNorms="true" />
|
|
||||||
<field name="name" type="string" indexed="true" stored="true" omitNorms="true" />
|
|
||||||
<field name="name_ngram" type="text_ngram" indexed="true" stored="true" />
|
|
||||||
<field name="revision_id" type="string" indexed="true" stored="true" omitNorms="true" />
|
|
||||||
<field name="version" type="string" indexed="true" stored="true" />
|
|
||||||
<field name="url" type="string" indexed="true" stored="true" omitNorms="true" />
|
|
||||||
<field name="ckan_url" type="string" indexed="true" stored="true" omitNorms="true" />
|
|
||||||
<field name="download_url" type="string" indexed="true" stored="true" omitNorms="true" />
|
|
||||||
<field name="notes" type="text" indexed="true" stored="true"/>
|
|
||||||
<field name="author" type="text_general" indexed="true" stored="true" />
|
|
||||||
<field name="author_email" type="text_general" indexed="true" stored="true" />
|
|
||||||
<field name="maintainer" type="text_general" indexed="true" stored="true" />
|
|
||||||
<field name="maintainer_email" type="text_general" indexed="true" stored="true" />
|
|
||||||
<field name="license" type="string" indexed="true" stored="true" />
|
|
||||||
<field name="license_id" type="string" indexed="true" stored="true" />
|
|
||||||
<field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
|
|
||||||
<field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
|
|
||||||
<field name="organization" type="string" indexed="true" stored="true" multiValued="false"/>
|
|
||||||
|
|
||||||
<field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
|
|
||||||
<field name="permission_labels" type="string" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
|
|
||||||
<field name="res_name" type="text_general" indexed="true" stored="true" multiValued="true" />
|
|
||||||
<field name="res_description" type="text_general" indexed="true" stored="true" multiValued="true"/>
|
|
||||||
<field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
|
|
||||||
<field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
|
|
||||||
<field name="res_type" type="string" indexed="true" stored="true" multiValued="true"/>
|
|
||||||
|
|
||||||
<!-- catchall field, containing all other searchable text fields (implemented
|
|
||||||
via copyField further on in this schema -->
|
|
||||||
<field name="text" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="urls" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
|
|
||||||
<field name="depends_on" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="dependency_of" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="derives_from" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="has_derivation" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="links_to" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
|
|
||||||
<field name="views_total" type="int" indexed="true" stored="false"/>
|
|
||||||
<field name="views_recent" type="int" indexed="true" stored="false"/>
|
|
||||||
<field name="resources_accessed_total" type="int" indexed="true" stored="false"/>
|
|
||||||
<field name="resources_accessed_recent" type="int" indexed="true" stored="false"/>
|
|
||||||
|
|
||||||
<field name="metadata_created" type="date" indexed="true" stored="true" multiValued="false"/>
|
|
||||||
<field name="metadata_modified" type="date" indexed="true" stored="true" multiValued="false"/>
|
|
||||||
|
|
||||||
<field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
|
|
||||||
|
|
||||||
<!-- Copy the title field into titleString, and treat as a string
|
|
||||||
(rather than text type). This allows us to sort on the titleString -->
|
|
||||||
<field name="title_string" type="string" indexed="true" stored="false" />
|
|
||||||
|
|
||||||
<field name="data_dict" type="string" indexed="false" stored="true" />
|
|
||||||
<field name="validated_data_dict" type="string" indexed="false" stored="true" />
|
|
||||||
|
|
||||||
<field name="_version_" type="string" indexed="true" stored="true"/>
|
|
||||||
|
|
||||||
<dynamicField name="*_date" type="date" indexed="true" stored="true" multiValued="false"/>
|
|
||||||
|
|
||||||
<dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
|
|
||||||
<dynamicField name="res_extras_*" type="text" indexed="true" stored="true" multiValued="true"/>
|
|
||||||
<dynamicField name="vocab_*" type="string" indexed="true" stored="true" multiValued="true"/>
|
|
||||||
<dynamicField name="*" type="string" indexed="true" stored="false"/>
|
|
||||||
</fields>
|
|
||||||
|
|
||||||
<uniqueKey>index_id</uniqueKey>
|
|
||||||
|
|
||||||
<copyField source="url" dest="urls"/>
|
|
||||||
<copyField source="title" dest="title_ngram"/>
|
|
||||||
<copyField source="name" dest="name_ngram"/>
|
|
||||||
<copyField source="ckan_url" dest="urls"/>
|
|
||||||
<copyField source="download_url" dest="urls"/>
|
|
||||||
<copyField source="res_url" dest="urls"/>
|
|
||||||
<copyField source="extras_*" dest="text"/>
|
|
||||||
<copyField source="res_extras_*" dest="text"/>
|
|
||||||
<copyField source="vocab_*" dest="text"/>
|
|
||||||
<copyField source="urls" dest="text"/>
|
|
||||||
<copyField source="name" dest="text"/>
|
|
||||||
<copyField source="title" dest="text"/>
|
|
||||||
<copyField source="text" dest="text"/>
|
|
||||||
<copyField source="license" dest="text"/>
|
|
||||||
<copyField source="notes" dest="text"/>
|
|
||||||
<copyField source="tags" dest="text"/>
|
|
||||||
<copyField source="groups" dest="text"/>
|
|
||||||
<copyField source="organization" dest="text"/>
|
|
||||||
<copyField source="res_name" dest="text"/>
|
|
||||||
<copyField source="res_description" dest="text"/>
|
|
||||||
<copyField source="maintainer" dest="text"/>
|
|
||||||
<copyField source="author" dest="text"/>
|
|
||||||
|
|
||||||
</schema>
|
|
||||||
|
|
@ -1,40 +0,0 @@
|
||||||
# Runtime configuration of CKAN enabled through ckanext-envvars
|
|
||||||
# Information about how it works: https://github.com/okfn/ckanext-envvars
|
|
||||||
# Note that variables here take presedence over build/up time variables in .env
|
|
||||||
|
|
||||||
# Set to true to disable CKAN from starting and serve a maintenance page
|
|
||||||
MAINTENANCE_MODE=false
|
|
||||||
|
|
||||||
# General Settings
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_MAX_UPLOAD_SIZE_MB=10
|
|
||||||
CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
|
|
||||||
# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
|
|
||||||
CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
|
|
||||||
CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
|
|
||||||
# CKAN Plugins
|
|
||||||
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
|
|
||||||
# CKAN requires storage path to be set in order for filestore to be enabled
|
|
||||||
CKAN__STORAGE_PATH=/srv/app/data
|
|
||||||
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
|
|
||||||
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
|
|
||||||
CKAN_SYSADMIN_NAME=sysadmin
|
|
||||||
CKAN_SYSADMIN_PASSWORD=password
|
|
||||||
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
|
|
||||||
|
|
||||||
# Email settings
|
|
||||||
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
|
|
||||||
CKAN_SMTP_STARTTLS=True
|
|
||||||
CKAN_SMTP_USER=user
|
|
||||||
CKAN_SMTP_PASSWORD=pass
|
|
||||||
CKAN_SMTP_MAIL_FROM=ckan@localhost
|
|
||||||
|
|
||||||
# Datapusher configuration
|
|
||||||
CKAN__DATAPUSHER__URL=http://datapusher:8000
|
|
||||||
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
|
|
||||||
|
|
||||||
# Solr configuration
|
|
||||||
CKAN_VERSION=2.9.11
|
|
||||||
CKAN_CORE_NAME=ckan
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
# Variables in this file will be used as build arguments when running
|
|
||||||
# docker-compose build and docker-compose up
|
|
||||||
# Verify correct substitution with "docker-compose config"
|
|
||||||
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
|
|
||||||
# docker-compose down -v
|
|
||||||
# docker-compose build
|
|
||||||
# docker-compose up -d
|
|
||||||
|
|
||||||
# Database
|
|
||||||
POSTGRES_PASSWORD=ckan
|
|
||||||
POSTGRES_PORT=5432
|
|
||||||
DATASTORE_READONLY_PASSWORD=datastore
|
|
||||||
|
|
||||||
# CKAN
|
|
||||||
CKAN_VERSION=2.9.11
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_MAX_UPLOAD_SIZE_MB=10
|
|
||||||
|
|
||||||
# Datapusher
|
|
||||||
DATAPUSHER_VERSION=0.0.17
|
|
||||||
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
|
|
||||||
DATAPUSHER_CHUNK_SIZE=16384
|
|
||||||
DATAPUSHER_CHUNK_INSERT_ROWS=250
|
|
||||||
DATAPUSHER_DOWNLOAD_TIMEOUT=30
|
|
||||||
DATAPUSHER_SSL_VERIFY=False
|
|
||||||
DATAPUSHER_REWRITE_RESOURCES=True
|
|
||||||
DATAPUSHER_REWRITE_URL=http://ckan:5000
|
|
||||||
|
|
||||||
# SOLR
|
|
||||||
CKAN_CORE_NAME=ckan
|
|
||||||
|
|
||||||
# Redis
|
|
||||||
REDIS_VERSION=6.0.7
|
|
||||||
|
|
@ -1,95 +0,0 @@
|
||||||
# docker-compose build && docker-compose up -d
|
|
||||||
version: "3"
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
ckan_data:
|
|
||||||
pg_data:
|
|
||||||
solr_data:
|
|
||||||
|
|
||||||
services:
|
|
||||||
ckan:
|
|
||||||
container_name: ckan
|
|
||||||
image: ghcr.io/keitaroinc/ckan:${CKAN_VERSION}
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- db
|
|
||||||
- solr
|
|
||||||
ports:
|
|
||||||
- "0.0.0.0:${CKAN_PORT}:5000"
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
|
||||||
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
|
||||||
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
|
||||||
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
- CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
|
||||||
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
volumes:
|
|
||||||
- ckan_data:/srv/app/data
|
|
||||||
|
|
||||||
datapusher:
|
|
||||||
container_name: datapusher
|
|
||||||
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
ports:
|
|
||||||
- "8000:8000"
|
|
||||||
environment:
|
|
||||||
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
|
|
||||||
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
|
|
||||||
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
|
|
||||||
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
|
|
||||||
- DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
|
|
||||||
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
|
|
||||||
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
|
|
||||||
|
|
||||||
|
|
||||||
db:
|
|
||||||
container_name: db
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: postgresql/Dockerfile
|
|
||||||
args:
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
environment:
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- PGDATA=/var/lib/postgresql/data/db
|
|
||||||
volumes:
|
|
||||||
- pg_data:/var/lib/postgresql/data
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "pg_isready", "-U", "ckan"]
|
|
||||||
|
|
||||||
solr:
|
|
||||||
container_name: solr
|
|
||||||
image: solr:8.11.1
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_CORE_NAME=${CKAN_CORE_NAME}
|
|
||||||
- CKAN_VERSION=${CKAN_VERSION}
|
|
||||||
volumes:
|
|
||||||
- solr_data:/var/solr
|
|
||||||
- ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
|
|
||||||
|
|
||||||
redis:
|
|
||||||
container_name: redis
|
|
||||||
image: redis:${REDIS_VERSION}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
|
|
||||||
networks:
|
|
||||||
frontend:
|
|
||||||
backend:
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
FROM postgis/postgis:14-3.2-alpine
|
|
||||||
|
|
||||||
# Allow connections; we don't map out any ports so only linked docker containers can connect
|
|
||||||
RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf
|
|
||||||
|
|
||||||
# Customize default user/pass/db
|
|
||||||
ENV POSTGRES_DB ckan
|
|
||||||
ENV POSTGRES_USER ckan
|
|
||||||
ARG POSTGRES_PASSWORD
|
|
||||||
ARG DS_RO_PASS
|
|
||||||
|
|
||||||
# Include datastore setup scripts
|
|
||||||
COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
|
|
||||||
CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
|
|
||||||
CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
|
|
||||||
GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;
|
|
||||||
EOSQL
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
ALTER VIEW geometry_columns OWNER TO ckan;
|
|
||||||
ALTER TABLE spatial_ref_sys OWNER TO ckan;
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
# Initialize SOLR for CKAN by creating a ckan core
|
|
||||||
# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION
|
|
||||||
# Example:
|
|
||||||
# CKAN_CORE_NAME=ckan
|
|
||||||
# CKAN_VERSION=2.9.11
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.solr8.xml
|
|
||||||
|
|
||||||
echo "Check whether managed schema exists for CKAN $CKAN_VERSION"
|
|
||||||
if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then
|
|
||||||
echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Check whether SOLR is initialized for CKAN"
|
|
||||||
CORESDIR=/var/solr/data
|
|
||||||
|
|
||||||
COREDIR="$CORESDIR/$CKAN_CORE_NAME"
|
|
||||||
if [ -d "$COREDIR" ]; then
|
|
||||||
echo "SOLR already initialized, skipping initialization"
|
|
||||||
else
|
|
||||||
echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION"
|
|
||||||
# init script for handling an empty /var/solr
|
|
||||||
/opt/docker-solr/scripts/init-var-solr
|
|
||||||
|
|
||||||
# Precreate CKAN core
|
|
||||||
/opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME
|
|
||||||
|
|
||||||
# Replace the managed schema with CKANs schema
|
|
||||||
echo "Adding CKAN managed schema"
|
|
||||||
curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s
|
|
||||||
|
|
||||||
echo "SOLR initialized"
|
|
||||||
fi
|
|
||||||
|
|
@ -1,37 +0,0 @@
|
||||||
# Runtime configuration of CKAN enabled through ckanext-envvars
|
|
||||||
# Information about how it works: https://github.com/okfn/ckanext-envvars
|
|
||||||
# Note that variables here take presedence over build/up time variables in .env
|
|
||||||
|
|
||||||
# General Settings
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_MAX_UPLOAD_SIZE_MB=10
|
|
||||||
# CKAN Plugins
|
|
||||||
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher harvest ckan_harvester
|
|
||||||
# CKAN requires storage path to be set in order for filestore to be enabled
|
|
||||||
CKAN__STORAGE_PATH=/srv/app/data
|
|
||||||
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
|
|
||||||
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
|
|
||||||
CKAN_SYSADMIN_NAME=sysadmin
|
|
||||||
CKAN_SYSADMIN_PASSWORD=password
|
|
||||||
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
|
|
||||||
|
|
||||||
# Email settings
|
|
||||||
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
|
|
||||||
CKAN_SMTP_STARTTLS=True
|
|
||||||
CKAN_SMTP_USER=user
|
|
||||||
CKAN_SMTP_PASSWORD=pass
|
|
||||||
CKAN_SMTP_MAIL_FROM=ckan@localhost
|
|
||||||
|
|
||||||
# Datapusher configuration
|
|
||||||
CKAN__DATAPUSHER__URL=http://datapusher:8000
|
|
||||||
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
|
|
||||||
|
|
||||||
# Harvest settings
|
|
||||||
CKAN__HARVEST__MQ__TYPE=redis
|
|
||||||
CKAN__HARVEST__MQ__HOSTNAME=redis
|
|
||||||
|
|
||||||
# Solr configuration
|
|
||||||
CKAN_VERSION=2.9.11
|
|
||||||
CKAN_CORE_NAME=ckan
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
# Variables in this file will be used as build arguments when running
|
|
||||||
# docker-compose build and docker-compose up
|
|
||||||
# Verify correct substitution with "docker-compose config"
|
|
||||||
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
|
|
||||||
# docker-compose down -v
|
|
||||||
# docker-compose build
|
|
||||||
# docker-compose up -d
|
|
||||||
|
|
||||||
# Database
|
|
||||||
POSTGRES_PASSWORD=ckan
|
|
||||||
POSTGRES_PORT=5432
|
|
||||||
DATASTORE_READONLY_PASSWORD=datastore
|
|
||||||
|
|
||||||
# CKAN
|
|
||||||
CKAN_VERSION=2.9.11
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_MAX_UPLOAD_SIZE_MB=10
|
|
||||||
|
|
||||||
# Datapusher
|
|
||||||
DATAPUSHER_VERSION=0.0.17
|
|
||||||
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
|
|
||||||
DATAPUSHER_CHUNK_SIZE=16384
|
|
||||||
DATAPUSHER_CHUNK_INSERT_ROWS=250
|
|
||||||
DATAPUSHER_DOWNLOAD_TIMEOUT=30
|
|
||||||
DATAPUSHER_SSL_VERIFY=False
|
|
||||||
DATAPUSHER_REWRITE_RESOURCES=True
|
|
||||||
DATAPUSHER_REWRITE_URL=http://ckan:5000
|
|
||||||
|
|
||||||
# SOLR
|
|
||||||
CKAN_CORE_NAME=ckan
|
|
||||||
|
|
||||||
# Redis
|
|
||||||
REDIS_VERSION=6.0.7
|
|
||||||
|
|
@ -1,58 +0,0 @@
|
||||||
###################
|
|
||||||
### Extensions ####
|
|
||||||
###################
|
|
||||||
FROM ghcr.io/keitaroinc/ckan:2.9.11 as extbuild
|
|
||||||
|
|
||||||
# Locations and tags, please use specific tags or revisions
|
|
||||||
ENV HARVEST_GIT_URL=https://github.com/ckan/ckanext-harvest
|
|
||||||
ENV HARVEST_GIT_BRANCH=v1.3.4
|
|
||||||
|
|
||||||
# Switch to the root user
|
|
||||||
USER root
|
|
||||||
|
|
||||||
# Install necessary packages to build extensions
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
python3-dev \
|
|
||||||
gcc \
|
|
||||||
g++ \
|
|
||||||
libffi-dev \
|
|
||||||
openssl-dev \
|
|
||||||
rust \
|
|
||||||
cargo
|
|
||||||
|
|
||||||
# Fetch and build the custom CKAN extensions
|
|
||||||
RUN pip wheel --wheel-dir=/wheels git+${HARVEST_GIT_URL}@${HARVEST_GIT_BRANCH}#egg=ckanext-harvest
|
|
||||||
RUN pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${HARVEST_GIT_BRANCH}/pip-requirements.txt
|
|
||||||
RUN curl -o /wheels/harvest.txt https://raw.githubusercontent.com/ckan/ckanext-harvest/${HARVEST_GIT_BRANCH}/pip-requirements.txt
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
||||||
############
|
|
||||||
### MAIN ###
|
|
||||||
############
|
|
||||||
FROM ghcr.io/keitaroinc/ckan:2.9.11
|
|
||||||
|
|
||||||
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
|
||||||
|
|
||||||
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher harvest ckan_harvester
|
|
||||||
|
|
||||||
# Switch to the root user
|
|
||||||
USER root
|
|
||||||
|
|
||||||
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Install and enable the custom extensions
|
|
||||||
RUN pip install --no-index --find-links=/srv/app/ext_wheels ckanext-harvest && \
|
|
||||||
pip install --no-index --find-links=/srv/app/ext_wheels -r /srv/app/ext_wheels/harvest.txt && \
|
|
||||||
# Configure plugins
|
|
||||||
ckan config-tool "${APP_DIR}/production.ini" "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
chown -R ckan:ckan /srv/app
|
|
||||||
|
|
||||||
# Remove wheels
|
|
||||||
RUN rm -rf /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Add harvest afterinit script
|
|
||||||
COPY ./afterinit.d/00_harvest.sh ${APP_DIR}/docker-afterinit.d/00_harvest.sh
|
|
||||||
|
|
||||||
# Switch to the ckan user
|
|
||||||
USER ckan
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
ckan -c /srv/app/production.ini harvester initdb
|
|
||||||
|
|
@ -1,192 +0,0 @@
|
||||||
# docker-compose build && docker-compose up -d
|
|
||||||
version: "3"
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
ckan_data:
|
|
||||||
pg_data:
|
|
||||||
solr_data:
|
|
||||||
|
|
||||||
services:
|
|
||||||
ckan:
|
|
||||||
container_name: ckan
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- db
|
|
||||||
ports:
|
|
||||||
- "0.0.0.0:${CKAN_PORT}:5000"
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
|
||||||
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
|
||||||
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
|
||||||
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
- CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
|
||||||
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
volumes:
|
|
||||||
- ckan_data:/srv/app/data
|
|
||||||
|
|
||||||
ckan-harvest-gather:
|
|
||||||
container_name: ckan-harvest-gather
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
command: ckan -c /srv/app/production.ini harvester gather-consumer
|
|
||||||
restart: on-failure
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- db
|
|
||||||
- ckan
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
|
||||||
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
|
||||||
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
|
||||||
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
- CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
|
||||||
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
|
|
||||||
ckan-harvest-fetch:
|
|
||||||
container_name: ckan-harvest-fetch
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
command: ckan -c /srv/app/production.ini harvester fetch-consumer
|
|
||||||
restart: on-failure
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- db
|
|
||||||
- ckan
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
|
||||||
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
|
||||||
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
|
||||||
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
- CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
|
||||||
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
|
|
||||||
ckan-harvest-run:
|
|
||||||
container_name: ckan-harvest-run
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
command: /bin/sh -c "while true; do sleep 900; ckan -c /srv/app/production.ini harvester run; done"
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- db
|
|
||||||
- ckan
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
|
||||||
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
|
||||||
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
|
||||||
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
- CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
|
||||||
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
|
|
||||||
ckan-harvest-cleanlog:
|
|
||||||
container_name: ckan-harvest-cleanlog
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
command: /bin/sh -c "while true; do sleep 86400; ckan -c /srv/app/production.ini harvester clean_harvest_log; done"
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- db
|
|
||||||
- ckan
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
|
||||||
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
|
||||||
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
|
||||||
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
- CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
|
||||||
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
|
|
||||||
datapusher:
|
|
||||||
container_name: datapusher
|
|
||||||
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
ports:
|
|
||||||
- "8000:8000"
|
|
||||||
environment:
|
|
||||||
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
|
|
||||||
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
|
|
||||||
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
|
|
||||||
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
|
|
||||||
- DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
|
|
||||||
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
|
|
||||||
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
|
|
||||||
|
|
||||||
db:
|
|
||||||
container_name: db
|
|
||||||
build:
|
|
||||||
context: ../../compose/2.9
|
|
||||||
dockerfile: postgresql/Dockerfile
|
|
||||||
args:
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
environment:
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- PGDATA=/var/lib/postgresql/data/db
|
|
||||||
volumes:
|
|
||||||
- pg_data:/var/lib/postgresql/data
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "pg_isready", "-U", "ckan"]
|
|
||||||
|
|
||||||
solr:
|
|
||||||
container_name: solr
|
|
||||||
image: solr:8.11.1
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_CORE_NAME=${CKAN_CORE_NAME}
|
|
||||||
- CKAN_VERSION=${CKAN_VERSION}
|
|
||||||
volumes:
|
|
||||||
- solr_data:/var/solr
|
|
||||||
- ${PWD}/../../compose/2.9/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
|
|
||||||
|
|
||||||
redis:
|
|
||||||
container_name: redis
|
|
||||||
image: redis:${REDIS_VERSION}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
|
|
||||||
networks:
|
|
||||||
frontend:
|
|
||||||
backend:
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
# Runtime configuration of CKAN enabled through ckanext-envvars
|
|
||||||
# Information about how it works: https://github.com/okfn/ckanext-envvars
|
|
||||||
# Note that variables here take presedence over build/up time variables in .env
|
|
||||||
|
|
||||||
# General Settings
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_MAX_UPLOAD_SIZE_MB=10
|
|
||||||
# CKAN Plugins
|
|
||||||
CKAN__PLUGINS=envvars s3filestore image_view webpage_view text_view recline_view datastore datapusher
|
|
||||||
# CKAN requires storage path to be set in order for filestore to be enabled
|
|
||||||
CKAN__STORAGE_PATH=/srv/app/data
|
|
||||||
CKAN__WEBASSETS__PATH=/srv/app/data/webassets
|
|
||||||
# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
|
|
||||||
CKAN_SYSADMIN_NAME=sysadmin
|
|
||||||
CKAN_SYSADMIN_PASSWORD=password
|
|
||||||
CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
|
|
||||||
CKAN__VIEWS__DEFAULT_VIEWS=image_view webpage_view text_view recline_view
|
|
||||||
|
|
||||||
# Email settings
|
|
||||||
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
|
|
||||||
CKAN_SMTP_STARTTLS=True
|
|
||||||
CKAN_SMTP_USER=user
|
|
||||||
CKAN_SMTP_PASSWORD=pass
|
|
||||||
CKAN_SMTP_MAIL_FROM=ckan@localhost
|
|
||||||
|
|
||||||
# Datapusher configuration
|
|
||||||
CKAN__DATAPUSHER__URL=http://datapusher:8000
|
|
||||||
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
|
|
||||||
|
|
||||||
# S3/MINIO settings
|
|
||||||
CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=MINIOACCESSKEY
|
|
||||||
CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=MINIOSECRETKEY
|
|
||||||
CKANEXT__S3FILESTORE__AWS_BUCKET_NAME=ckan
|
|
||||||
CKANEXT__S3FILESTORE__HOST_NAME=http://minio:9000
|
|
||||||
CKANEXT__S3FILESTORE__REGION_NAME=us-east-1
|
|
||||||
CKANEXT__S3FILESTORE__SIGNATURE_VERSION=s3v4
|
|
||||||
|
|
||||||
# Solr configuration
|
|
||||||
CKAN_VERSION=2.9.11
|
|
||||||
CKAN_CORE_NAME=ckan
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
# Variables in this file will be used as build arguments when running
|
|
||||||
# docker-compose build and docker-compose up
|
|
||||||
# Verify correct substitution with "docker-compose config"
|
|
||||||
# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
|
|
||||||
# docker-compose down -v
|
|
||||||
# docker-compose build
|
|
||||||
# docker-compose up -d
|
|
||||||
|
|
||||||
# Database
|
|
||||||
POSTGRES_PASSWORD=ckan
|
|
||||||
POSTGRES_PORT=5432
|
|
||||||
DATASTORE_READONLY_PASSWORD=datastore
|
|
||||||
|
|
||||||
# CKAN
|
|
||||||
CKAN_VERSION=2.9.11
|
|
||||||
CKAN_SITE_ID=default
|
|
||||||
CKAN_SITE_URL=http://localhost:5000
|
|
||||||
CKAN_PORT=5000
|
|
||||||
CKAN_MAX_UPLOAD_SIZE_MB=10
|
|
||||||
|
|
||||||
# Datapusher
|
|
||||||
DATAPUSHER_VERSION=0.0.17
|
|
||||||
DATAPUSHER_MAX_CONTENT_LENGTH=10485760
|
|
||||||
DATAPUSHER_CHUNK_SIZE=16384
|
|
||||||
DATAPUSHER_CHUNK_INSERT_ROWS=250
|
|
||||||
DATAPUSHER_DOWNLOAD_TIMEOUT=30
|
|
||||||
DATAPUSHER_SSL_VERIFY=False
|
|
||||||
DATAPUSHER_REWRITE_RESOURCES=True
|
|
||||||
DATAPUSHER_REWRITE_URL=http://ckan:5000
|
|
||||||
|
|
||||||
# SOLR
|
|
||||||
CKAN_CORE_NAME=ckan
|
|
||||||
|
|
||||||
# Redis
|
|
||||||
REDIS_VERSION=6.0.7
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
###################
|
|
||||||
### Extensions ####
|
|
||||||
###################
|
|
||||||
FROM ghcr.io/keitaroinc/ckan:2.9.11 as extbuild
|
|
||||||
|
|
||||||
# Locations and tags, please use specific tags or revisions
|
|
||||||
ENV S3FILESTORE_GIT_URL=https://github.com/keitaroinc/ckanext-s3filestore
|
|
||||||
ENV S3FILESTORE_GIT_BRANCH=v1.0.0
|
|
||||||
|
|
||||||
# Switch to the root user
|
|
||||||
USER root
|
|
||||||
|
|
||||||
# Fetch and build the custom CKAN extensions
|
|
||||||
RUN pip wheel --wheel-dir=/wheels git+${S3FILESTORE_GIT_URL}@${S3FILESTORE_GIT_BRANCH}#egg=ckanext-s3filestore
|
|
||||||
RUN pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/keitaroinc/ckanext-s3filestore/${S3FILESTORE_GIT_BRANCH}/requirements.txt
|
|
||||||
RUN curl -o /wheels/s3filestore.txt https://raw.githubusercontent.com/keitaroinc/ckanext-s3filestore/${S3FILESTORE_GIT_BRANCH}/requirements.txt
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
||||||
############
|
|
||||||
### MAIN ###
|
|
||||||
############
|
|
||||||
FROM ghcr.io/keitaroinc/ckan:2.9.11
|
|
||||||
|
|
||||||
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
|
||||||
|
|
||||||
ENV CKAN__PLUGINS envvars s3filestore image_view webpage_view text_view recline_view datastore datapusher
|
|
||||||
|
|
||||||
# Switch to the root user
|
|
||||||
USER root
|
|
||||||
|
|
||||||
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Install and enable the custom extensions
|
|
||||||
RUN pip install --no-index --find-links=/srv/app/ext_wheels ckanext-s3filestore && \
|
|
||||||
pip install --no-index --find-links=/srv/app/ext_wheels -r /srv/app/ext_wheels/s3filestore.txt && \
|
|
||||||
ckan config-tool "${APP_DIR}/production.ini" "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
chown -R ckan:ckan /srv/app
|
|
||||||
|
|
||||||
# Remove wheels
|
|
||||||
RUN rm -rf /srv/app/ext_wheels
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
@ -1,122 +0,0 @@
|
||||||
# docker-compose build && docker-compose up -d
|
|
||||||
version: "3"
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
minio_data:
|
|
||||||
pg_data:
|
|
||||||
solr_data:
|
|
||||||
|
|
||||||
services:
|
|
||||||
ckan:
|
|
||||||
container_name: ckan
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- db
|
|
||||||
ports:
|
|
||||||
- "0.0.0.0:${CKAN_PORT}:5000"
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
|
|
||||||
- CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
|
|
||||||
- CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
|
|
||||||
- CKAN_SOLR_URL=http://solr:8983/solr/ckan
|
|
||||||
- CKAN_REDIS_URL=redis://redis:6379/1
|
|
||||||
- CKAN_SITE_URL=${CKAN_SITE_URL}
|
|
||||||
- CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
|
|
||||||
datapusher:
|
|
||||||
container_name: datapusher
|
|
||||||
image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION}
|
|
||||||
networks:
|
|
||||||
- frontend
|
|
||||||
- backend
|
|
||||||
ports:
|
|
||||||
- "8000:8000"
|
|
||||||
environment:
|
|
||||||
- DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
|
|
||||||
- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
|
|
||||||
- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
|
|
||||||
- DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
|
|
||||||
- DATAPUSHER_SSL_VERIFY=${DATAPUSHER_SSL_VERIFY}
|
|
||||||
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
|
|
||||||
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
|
|
||||||
|
|
||||||
db:
|
|
||||||
container_name: db
|
|
||||||
build:
|
|
||||||
context: ../../compose/2.9
|
|
||||||
dockerfile: postgresql/Dockerfile
|
|
||||||
args:
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
environment:
|
|
||||||
- DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
|
|
||||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
||||||
- PGDATA=/var/lib/postgresql/data/db
|
|
||||||
volumes:
|
|
||||||
- pg_data:/var/lib/postgresql/data
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "pg_isready", "-U", "ckan"]
|
|
||||||
|
|
||||||
solr:
|
|
||||||
container_name: solr
|
|
||||||
image: solr:8.11.1
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
env_file:
|
|
||||||
- ./.ckan-env
|
|
||||||
environment:
|
|
||||||
- CKAN_CORE_NAME=${CKAN_CORE_NAME}
|
|
||||||
- CKAN_VERSION=${CKAN_VERSION}
|
|
||||||
volumes:
|
|
||||||
- solr_data:/var/solr
|
|
||||||
- ${PWD}/../../compose/2.9/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh
|
|
||||||
|
|
||||||
redis:
|
|
||||||
container_name: redis
|
|
||||||
image: redis:${REDIS_VERSION}
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
|
|
||||||
minio:
|
|
||||||
container_name: minio
|
|
||||||
image: minio/minio:RELEASE.2020-08-08T04-50-06Z
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
- frontend
|
|
||||||
ports:
|
|
||||||
- "0.0.0.0:9000:9000"
|
|
||||||
environment:
|
|
||||||
- MINIO_ACCESS_KEY=MINIOACCESSKEY
|
|
||||||
- MINIO_SECRET_KEY=MINIOSECRETKEY
|
|
||||||
volumes:
|
|
||||||
- minio_data:/data
|
|
||||||
command: server /data
|
|
||||||
|
|
||||||
mc:
|
|
||||||
container_name: mc
|
|
||||||
image: minio/mc:RELEASE.2020-08-08T02-33-58Z
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
- minio
|
|
||||||
entrypoint: >
|
|
||||||
/bin/sh -c "
|
|
||||||
/usr/bin/mc config host rm local;
|
|
||||||
/usr/bin/mc config host add --api s3v4 local http://minio:9000 MINIOACCESSKEY MINIOSECRETKEY;
|
|
||||||
/usr/bin/mc mb local/ckan/;
|
|
||||||
/usr/bin/mc policy set download local/ckan/storage;
|
|
||||||
"
|
|
||||||
|
|
||||||
networks:
|
|
||||||
frontend:
|
|
||||||
backend:
|
|
||||||
|
|
@ -1,196 +0,0 @@
|
||||||
##################
|
|
||||||
### Build CKAN ###
|
|
||||||
##################
|
|
||||||
FROM alpine:3.14.2 as ckanbuild
|
|
||||||
|
|
||||||
# Used by Github Actions to tag the image with
|
|
||||||
ENV IMAGE_TAG=2.7.12
|
|
||||||
|
|
||||||
# Set CKAN version to build
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
ENV GIT_BRANCH=ckan-2.7.12
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${SRC_DIR}
|
|
||||||
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
python2 \
|
|
||||||
postgresql-dev \
|
|
||||||
linux-headers \
|
|
||||||
gcc \
|
|
||||||
make \
|
|
||||||
g++ \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
libtool \
|
|
||||||
patch \
|
|
||||||
musl-dev \
|
|
||||||
pcre-dev \
|
|
||||||
pcre \
|
|
||||||
python2-dev \
|
|
||||||
libffi-dev \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev
|
|
||||||
|
|
||||||
# Install version 9.x of postgresql-dev so that CKAN requirements can be built
|
|
||||||
RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.6/main \
|
|
||||||
postgresql-dev~=9.6
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Fetch and build CKAN and requirements
|
|
||||||
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
|
||||||
# Copy patches and apply patches script
|
|
||||||
COPY ./patches ${SRC_DIR}/patches
|
|
||||||
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
# Apply patches
|
|
||||||
RUN ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
RUN rm -rf /srv/app/src/ckan/.git
|
|
||||||
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
|
|
||||||
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==21.12.0 greenlet==1.1.3
|
|
||||||
|
|
||||||
|
|
||||||
###########################
|
|
||||||
### Default-Extensions ####
|
|
||||||
###########################
|
|
||||||
FROM alpine:3.14.2 as extbuild
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
# List of default extensions
|
|
||||||
ENV DEFAULT_EXTENSIONS envvars
|
|
||||||
|
|
||||||
# Locations and tags, please use specific tags or revisions
|
|
||||||
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
|
|
||||||
ENV ENVVARS_GIT_BRANCH=0.0.1
|
|
||||||
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
python2 \
|
|
||||||
python2-dev
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Fetch and build the default CKAN extensions
|
|
||||||
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
|
|
||||||
|
|
||||||
############
|
|
||||||
### MAIN ###
|
|
||||||
############
|
|
||||||
FROM alpine:3.14.2
|
|
||||||
|
|
||||||
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
|
||||||
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_DIR=${SRC_DIR}/ckan
|
|
||||||
ENV DATA_DIR=/srv/app/data
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
git \
|
|
||||||
bash \
|
|
||||||
git \
|
|
||||||
gettext \
|
|
||||||
curl \
|
|
||||||
python2 \
|
|
||||||
libmagic \
|
|
||||||
pcre \
|
|
||||||
libxslt \
|
|
||||||
libxml2 \
|
|
||||||
tzdata \
|
|
||||||
apache2-utils && \
|
|
||||||
# Install version 9.x of postgresql-client so that CKAN can run
|
|
||||||
apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.6/main \
|
|
||||||
postgresql-client~=9.6 && \
|
|
||||||
# Create SRC_DIR
|
|
||||||
mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Get artifacts from build stages
|
|
||||||
COPY --from=ckanbuild /wheels /srv/app/wheels
|
|
||||||
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
|
||||||
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Additional install steps for build stages artifacts
|
|
||||||
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi==2.0.19.1 gevent==21.12.0
|
|
||||||
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN addgroup -g 92 -S ckan && \
|
|
||||||
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
|
||||||
|
|
||||||
WORKDIR ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Install CKAN
|
|
||||||
RUN pip install -e /srv/app/src/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
|
|
||||||
# Install default CKAN extensions
|
|
||||||
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
|
|
||||||
# Create and update CKAN config
|
|
||||||
# Set timezone
|
|
||||||
echo "UTC" > /etc/timezone && \
|
|
||||||
# Generate CKAN config
|
|
||||||
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
# Set the default level for extensions to INFO
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
|
|
||||||
# Create the data directory
|
|
||||||
mkdir ${DATA_DIR} && \
|
|
||||||
# Change ownership to app user
|
|
||||||
chown -R ckan:ckan /srv/app
|
|
||||||
|
|
||||||
# Remove wheels
|
|
||||||
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Install python2 secrets for generating sessions
|
|
||||||
RUN pip install python2-secrets
|
|
||||||
|
|
||||||
# Copy necessary scripts
|
|
||||||
COPY setup/app ${APP_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Copy the alias script for paster to be ckan so it's compatible with 2.9
|
|
||||||
COPY setup/bin/ckan /usr/bin/ckan
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-entrypoint.d
|
|
||||||
|
|
||||||
# Create afterinit directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-afterinit.d
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,240 +0,0 @@
|
||||||
##################
|
|
||||||
### Build CKAN ###
|
|
||||||
##################
|
|
||||||
FROM ubuntu:focal-20210827 as ckanbuild
|
|
||||||
|
|
||||||
# Used by Github Actions to tag the image with
|
|
||||||
ENV IMAGE_TAG=2.7.12-focal
|
|
||||||
|
|
||||||
# Set CKAN version to build
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
ENV GIT_BRANCH=ckan-2.7.12
|
|
||||||
|
|
||||||
# Set timezone
|
|
||||||
ENV TZ=UTC
|
|
||||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
|
||||||
|
|
||||||
# Set Locale
|
|
||||||
ENV LC_ALL=en_US.UTF-8
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${SRC_DIR}
|
|
||||||
|
|
||||||
# Set the locale
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get install --no-install-recommends -y locales
|
|
||||||
RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
|
|
||||||
RUN dpkg-reconfigure --frontend=noninteractive locales
|
|
||||||
RUN update-locale LANG=${LC_ALL}
|
|
||||||
|
|
||||||
# Instal apt-utils
|
|
||||||
RUN apt-get install --no-install-recommends -y \
|
|
||||||
apt-utils
|
|
||||||
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
RUN apt-get install --no-install-recommends -y \
|
|
||||||
git \
|
|
||||||
gnupg \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
python \
|
|
||||||
linux-headers-generic \
|
|
||||||
gcc-10 \
|
|
||||||
make \
|
|
||||||
g++-10 \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
libtool \
|
|
||||||
patch \
|
|
||||||
libpcre3-dev \
|
|
||||||
libpcre3 \
|
|
||||||
python-dev \
|
|
||||||
libffi-dev \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev
|
|
||||||
|
|
||||||
# Install version 9.x of postgresql-dev so that CKAN requirements can be built
|
|
||||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
|
||||||
RUN curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
|
|
||||||
RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" >> /etc/apt/sources.list.d/postgresql.list'
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get install --no-install-recommends -y \
|
|
||||||
postgresql-server-dev-9.6
|
|
||||||
|
|
||||||
# Use gcc 10
|
|
||||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10
|
|
||||||
|
|
||||||
#postgresql-server-dev-9.6 Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Fetch and build CKAN and requirements
|
|
||||||
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
|
||||||
# Copy patches and apply patches script
|
|
||||||
COPY ./patches ${SRC_DIR}/patches
|
|
||||||
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
# Apply patches
|
|
||||||
RUN ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
RUN rm -rf /srv/app/src/ckan/.git
|
|
||||||
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
|
|
||||||
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==21.12.0 greenlet==1.1.3
|
|
||||||
|
|
||||||
|
|
||||||
###########################
|
|
||||||
### Default-Extensions ####
|
|
||||||
###########################
|
|
||||||
FROM ubuntu:focal-20210827 as extbuild
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
# List of default extensions
|
|
||||||
ENV DEFAULT_EXTENSIONS envvars
|
|
||||||
|
|
||||||
# Locations and tags, please use specific tags or revisions
|
|
||||||
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
|
|
||||||
ENV ENVVARS_GIT_BRANCH=0.0.1
|
|
||||||
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
python \
|
|
||||||
python-dev
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Fetch and build the default CKAN extensions
|
|
||||||
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
|
|
||||||
|
|
||||||
############
|
|
||||||
### MAIN ###
|
|
||||||
############
|
|
||||||
FROM ubuntu:focal-20210827
|
|
||||||
|
|
||||||
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
|
||||||
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
|
|
||||||
|
|
||||||
# Set timezone
|
|
||||||
ENV TZ=UTC
|
|
||||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
|
||||||
|
|
||||||
# Set Locale
|
|
||||||
ENV LC_ALL=en_US.UTF-8
|
|
||||||
|
|
||||||
# Set the locale
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y locales && \
|
|
||||||
sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \
|
|
||||||
dpkg-reconfigure --frontend=noninteractive locales && \
|
|
||||||
update-locale LANG=${LC_ALL} && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_DIR=${SRC_DIR}/ckan
|
|
||||||
ENV DATA_DIR=/srv/app/data
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y \
|
|
||||||
gettext \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
git \
|
|
||||||
gnupg \
|
|
||||||
python \
|
|
||||||
libpython2.7 \
|
|
||||||
libmagic1 \
|
|
||||||
libpcre3 \
|
|
||||||
libxslt1.1 \
|
|
||||||
libxml2 \
|
|
||||||
tzdata \
|
|
||||||
apache2-utils && \
|
|
||||||
rm -rf /var/lib/apt/lists/* && \
|
|
||||||
# Install version 9.x of postgresql-client so that CKAN can run
|
|
||||||
curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
|
|
||||||
sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" >> /etc/apt/sources.list.d/postgresql.list' && \
|
|
||||||
apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y \
|
|
||||||
postgresql-client-9.6 && \
|
|
||||||
rm -rf /var/lib/apt/lists/* && \
|
|
||||||
# Create SRC_DIR
|
|
||||||
mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Get artifacts from build stages
|
|
||||||
COPY --from=ckanbuild /wheels /srv/app/wheels
|
|
||||||
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
|
||||||
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Additional install steps for build stages artifacts
|
|
||||||
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi==2.0.19.1 gevent==21.12.0
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN groupadd -g 92 ckan && \
|
|
||||||
useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan
|
|
||||||
|
|
||||||
WORKDIR ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Install CKAN
|
|
||||||
RUN pip install -e /srv/app/src/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
|
|
||||||
# Install default CKAN extensions
|
|
||||||
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
|
|
||||||
# Create and update CKAN config
|
|
||||||
# Generate CKAN config
|
|
||||||
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
# Set the default level for extensions to INFO
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
|
|
||||||
# Create the data directory
|
|
||||||
mkdir ${DATA_DIR} && \
|
|
||||||
# Change ownership to app user
|
|
||||||
chown -R ckan:ckan /srv/app
|
|
||||||
|
|
||||||
# Remove wheels
|
|
||||||
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Copy necessary scripts
|
|
||||||
COPY setup/app ${APP_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Copy the alias script for paster to be ckan so it's compatible with 2.9
|
|
||||||
COPY setup/bin/ckan /usr/bin/ckan
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-entrypoint.d
|
|
||||||
|
|
||||||
# Create afterinit directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-afterinit.d
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
shopt -s nullglob
|
|
||||||
for patch in patches/*.patch; do
|
|
||||||
/usr/bin/patch -p0 -i $patch
|
|
||||||
done
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# this is called before uwsgi is executed
|
|
||||||
# uset his to add extra scripts before ckan is started
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
<!--
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Maintenance</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Maintenance</h1>
|
|
||||||
<p>Our data portal is currently in maintenance, please try in a while.</p>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
<!--
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Maintenance</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Maintenance</h1>
|
|
||||||
<p>Our data portal is currently in maintenance, please try in a while.</p>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
"""
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from SimpleHTTPServer import SimpleHTTPRequestHandler
|
|
||||||
from BaseHTTPServer import HTTPServer
|
|
||||||
from SocketServer import ThreadingMixIn
|
|
||||||
import os
|
|
||||||
|
|
||||||
PORT = 5000
|
|
||||||
|
|
||||||
web_dir = os.path.join(os.path.dirname(__file__))
|
|
||||||
os.chdir(web_dir)
|
|
||||||
|
|
||||||
Handler = SimpleHTTPRequestHandler
|
|
||||||
|
|
||||||
|
|
||||||
class MaintenanceServer(ThreadingMixIn, HTTPServer):
|
|
||||||
"""Handle requests in a separate thread."""
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
httpd = MaintenanceServer(("0.0.0.0", PORT), Handler)
|
|
||||||
print("Starting maintenance mode")
|
|
||||||
httpd.serve_forever()
|
|
||||||
|
|
||||||
|
|
@ -1,200 +0,0 @@
|
||||||
"""
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import subprocess
|
|
||||||
import psycopg2
|
|
||||||
import urllib2
|
|
||||||
import re
|
|
||||||
|
|
||||||
import time
|
|
||||||
|
|
||||||
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
|
|
||||||
|
|
||||||
RETRY = 5
|
|
||||||
|
|
||||||
def check_db_connection(retry=None):
|
|
||||||
|
|
||||||
print('[prerun] Start check_db_connection...')
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
|
|
||||||
try:
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print((str(e)))
|
|
||||||
print('[prerun] Unable to connect to the database...try again in a while.')
|
|
||||||
import time
|
|
||||||
time.sleep(10)
|
|
||||||
check_db_connection(retry = retry - 1)
|
|
||||||
else:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
def check_solr_connection(retry=None):
|
|
||||||
|
|
||||||
print('[prerun] Start check_solr_connection...')
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
url = os.environ.get('CKAN_SOLR_URL', '')
|
|
||||||
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection = urllib2.urlopen(search_url)
|
|
||||||
except urllib2.URLError as e:
|
|
||||||
print('[prerun] Unable to connect to solr...try again in a while.')
|
|
||||||
import time
|
|
||||||
time.sleep(10)
|
|
||||||
check_solr_connection(retry = retry - 1)
|
|
||||||
else:
|
|
||||||
import re
|
|
||||||
conn_info = connection.read()
|
|
||||||
# SolrCloud
|
|
||||||
conn_info = re.sub(r'"zkConnected":true', '"zkConnected":True', conn_info)
|
|
||||||
eval(conn_info)
|
|
||||||
|
|
||||||
def init_db():
|
|
||||||
|
|
||||||
print('[prerun] Start init_db...')
|
|
||||||
|
|
||||||
db_command = ['paster', '--plugin=ckan', 'db', 'init', '-c', ckan_ini]
|
|
||||||
|
|
||||||
print('[prerun] Initializing or upgrading db - start using paster db init')
|
|
||||||
try:
|
|
||||||
# run init scripts
|
|
||||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
|
||||||
|
|
||||||
print('[prerun] Initializing or upgrading db - end')
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
import time
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print((e.output))
|
|
||||||
raise e
|
|
||||||
print('[prerun] Initializing or upgrading db - finish')
|
|
||||||
|
|
||||||
|
|
||||||
def init_datastore():
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] Skipping datastore initialization')
|
|
||||||
return
|
|
||||||
|
|
||||||
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
|
|
||||||
'set-permissions', '-c', ckan_ini]
|
|
||||||
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
cursor = connection.cursor()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - start')
|
|
||||||
try:
|
|
||||||
datastore_perms = subprocess.Popen(
|
|
||||||
datastore_perms_command,
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
perms_sql = datastore_perms.stdout.read()
|
|
||||||
# Remove internal pg command as psycopg2 does not like it
|
|
||||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql.decode('utf-8'))
|
|
||||||
cursor.execute(perms_sql)
|
|
||||||
for notice in connection.notices:
|
|
||||||
print(notice)
|
|
||||||
|
|
||||||
connection.commit()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - end')
|
|
||||||
print((datastore_perms.stdout.read()))
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print('[prerun] Could not initialize datastore')
|
|
||||||
print((str(e)))
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print((e.output))
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print((e.output))
|
|
||||||
raise e
|
|
||||||
finally:
|
|
||||||
cursor.close()
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def create_sysadmin():
|
|
||||||
|
|
||||||
print('[prerun] Start create_sysadmin...')
|
|
||||||
|
|
||||||
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
|
||||||
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
|
||||||
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
|
||||||
|
|
||||||
if name and password and email:
|
|
||||||
|
|
||||||
# Check if user exists
|
|
||||||
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
|
|
||||||
|
|
||||||
out = subprocess.check_output(command)
|
|
||||||
if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')):
|
|
||||||
print('[prerun] Sysadmin user exists, skipping creation')
|
|
||||||
return
|
|
||||||
|
|
||||||
# Create user
|
|
||||||
command = ['paster', '--plugin=ckan', 'user', 'add',
|
|
||||||
name,
|
|
||||||
'password=' + password,
|
|
||||||
'email=' + email,
|
|
||||||
'-c', ckan_ini]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print(('[prerun] Created user {0}'.format(name)))
|
|
||||||
|
|
||||||
# Make it sysadmin
|
|
||||||
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
|
|
||||||
name,
|
|
||||||
'-c', ckan_ini]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print(('[prerun] Made user {0} a sysadmin'.format(name)))
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
|
||||||
|
|
||||||
if maintenance:
|
|
||||||
print('[prerun] Maintenance mode, skipping setup...')
|
|
||||||
else:
|
|
||||||
check_db_connection()
|
|
||||||
check_solr_connection()
|
|
||||||
init_db()
|
|
||||||
if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
|
|
||||||
init_datastore()
|
|
||||||
create_sysadmin()
|
|
||||||
|
|
@ -1,74 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
# Run any startup scripts provided by images extending this one
|
|
||||||
if [[ -d "${APP_DIR}/docker-entrypoint.d" ]]
|
|
||||||
then
|
|
||||||
for f in ${APP_DIR}/docker-entrypoint.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add session secret from chart
|
|
||||||
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET ]];then
|
|
||||||
echo "Not all environment variables are set. Generating sessions..."
|
|
||||||
else
|
|
||||||
echo "Setting session secrets from environment variables"
|
|
||||||
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
|
|
||||||
then
|
|
||||||
echo "Setting beaker.session.secret in ini file"
|
|
||||||
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python -c 'import secrets; print(secrets.token_urlsafe())')"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Set the common uwsgi options
|
|
||||||
echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
|
|
||||||
|
|
||||||
UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger /srv/app/production.ini --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch"
|
|
||||||
|
|
||||||
# Run the prerun script to init CKAN and create the default admin user
|
|
||||||
python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }
|
|
||||||
|
|
||||||
# Check if we are in maintenance mode and if yes serve the maintenance pages
|
|
||||||
if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi
|
|
||||||
|
|
||||||
# Run any after prerun/init scripts provided by images extending this one
|
|
||||||
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
|
|
||||||
then
|
|
||||||
for f in ${APP_DIR}/docker-afterinit.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
|
||||||
if [ $? -eq 0 ]
|
|
||||||
then
|
|
||||||
if [ "$PASSWORD_PROTECT" = true ]
|
|
||||||
then
|
|
||||||
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
|
||||||
then
|
|
||||||
# Generate htpasswd file for basicauth
|
|
||||||
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
|
||||||
# Start uwsgi with basicauth
|
|
||||||
uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
|
|
||||||
else
|
|
||||||
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
# Start uwsgi
|
|
||||||
uwsgi $UWSGI_OPTS
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "[prerun] failed...not starting CKAN."
|
|
||||||
fi
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
# CKAN <2.9 uses paster and the ckan plugin for paster which expects the configuration to be passed
|
|
||||||
# to a subcommand, example: paster --plugin=ckan datastore -c /srv/app/production.ini set-permissions
|
|
||||||
# CKAN >2.9 ckan CLI command expects the configuration option as the second and third parameter
|
|
||||||
# Example: ckan -c /srv/app/production.ini datastore set-permissions
|
|
||||||
#
|
|
||||||
# Shift three arguments so that we can reorder the argument list
|
|
||||||
# ckan -c /srv/app/production.ini datastore set-permissions
|
|
||||||
# becomes
|
|
||||||
# paster --plugin=ckan datastore -c /srv/app/production.ini set-permissions
|
|
||||||
config_switch=$1
|
|
||||||
config_file=$2
|
|
||||||
subcommand=$3
|
|
||||||
shift 3
|
|
||||||
paster --plugin=ckan "$subcommand" "$config_switch" "$config_file" "$@"
|
|
||||||
|
|
@ -1,189 +0,0 @@
|
||||||
##################
|
|
||||||
### Build CKAN ###
|
|
||||||
##################
|
|
||||||
FROM alpine:3.14.2 as ckanbuild
|
|
||||||
|
|
||||||
# Used by Github Actions to tag the image with
|
|
||||||
ENV IMAGE_TAG=2.8.12
|
|
||||||
|
|
||||||
# Set CKAN version to build
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
ENV GIT_BRANCH=ckan-2.8.12
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${SRC_DIR}
|
|
||||||
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
python2 \
|
|
||||||
postgresql-dev \
|
|
||||||
linux-headers \
|
|
||||||
gcc \
|
|
||||||
make \
|
|
||||||
g++ \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
patch \
|
|
||||||
libtool \
|
|
||||||
musl-dev \
|
|
||||||
pcre-dev \
|
|
||||||
pcre \
|
|
||||||
python2-dev \
|
|
||||||
libffi-dev \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Fetch and build CKAN and requirements
|
|
||||||
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
|
||||||
# Copy patches and apply patches script
|
|
||||||
COPY ./patches ${SRC_DIR}/patches
|
|
||||||
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
# Apply patches
|
|
||||||
RUN ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
RUN rm -rf /srv/app/src/ckan/.git
|
|
||||||
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
|
|
||||||
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==21.12.0 greenlet==1.1.3
|
|
||||||
|
|
||||||
|
|
||||||
###########################
|
|
||||||
### Default-Extensions ####
|
|
||||||
###########################
|
|
||||||
FROM alpine:3.14.2 as extbuild
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
# List of default extensions
|
|
||||||
ENV DEFAULT_EXTENSIONS envvars
|
|
||||||
|
|
||||||
# Locations and tags, please use specific tags or revisions
|
|
||||||
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
|
|
||||||
ENV ENVVARS_GIT_BRANCH=0.0.1
|
|
||||||
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
python2 \
|
|
||||||
python2-dev
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Fetch and build the default CKAN extensions
|
|
||||||
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
|
|
||||||
|
|
||||||
############
|
|
||||||
### MAIN ###
|
|
||||||
############
|
|
||||||
FROM alpine:3.14.2
|
|
||||||
|
|
||||||
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
|
||||||
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_DIR=${SRC_DIR}/ckan
|
|
||||||
ENV DATA_DIR=/srv/app/data
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
RUN apk add --no-cache git \
|
|
||||||
bash \
|
|
||||||
git \
|
|
||||||
gettext \
|
|
||||||
curl \
|
|
||||||
postgresql-client \
|
|
||||||
python2 \
|
|
||||||
libmagic \
|
|
||||||
pcre \
|
|
||||||
libxslt \
|
|
||||||
libxml2 \
|
|
||||||
tzdata \
|
|
||||||
apache2-utils && \
|
|
||||||
# Create SRC_DIR
|
|
||||||
mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Get artifacts from build stages
|
|
||||||
COPY --from=ckanbuild /wheels /srv/app/wheels
|
|
||||||
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
|
||||||
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Additional install steps for build stages artifacts
|
|
||||||
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi==2.0.19.1 gevent==21.12.0
|
|
||||||
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN addgroup -g 92 -S ckan && \
|
|
||||||
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
|
||||||
|
|
||||||
WORKDIR ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Install CKAN
|
|
||||||
RUN pip install -e /srv/app/src/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
|
|
||||||
# Install default CKAN extensions
|
|
||||||
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
|
|
||||||
# Create and update CKAN config
|
|
||||||
# Set timezone
|
|
||||||
echo "UTC" > /etc/timezone && \
|
|
||||||
# Generate CKAN config
|
|
||||||
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
# Set the default level for extensions to INFO
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
|
|
||||||
# Create the data directory
|
|
||||||
mkdir ${DATA_DIR} && \
|
|
||||||
# Change ownership to app user
|
|
||||||
chown -R ckan:ckan /srv/app
|
|
||||||
|
|
||||||
# Remove wheels
|
|
||||||
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Install python2 secrets for generating sessions
|
|
||||||
RUN pip install python2-secrets
|
|
||||||
|
|
||||||
# Copy necessary scripts
|
|
||||||
COPY setup/app ${APP_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Copy the alias script for paster to be ckan so it's compatible with 2.9
|
|
||||||
COPY setup/bin/ckan /usr/bin/ckan
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-entrypoint.d
|
|
||||||
|
|
||||||
# Create afterinit directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-afterinit.d
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,226 +0,0 @@
|
||||||
##################
|
|
||||||
### Build CKAN ###
|
|
||||||
##################
|
|
||||||
FROM ubuntu:focal-20210827 as ckanbuild
|
|
||||||
|
|
||||||
# Used by Github Actions to tag the image with
|
|
||||||
ENV IMAGE_TAG=2.8.12-focal
|
|
||||||
|
|
||||||
# Set CKAN version to build
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
ENV GIT_BRANCH=ckan-2.8.12
|
|
||||||
|
|
||||||
# Set timezone
|
|
||||||
ENV TZ=UTC
|
|
||||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
|
||||||
|
|
||||||
# Set Locale
|
|
||||||
ENV LC_ALL=en_US.UTF-8
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${SRC_DIR}
|
|
||||||
|
|
||||||
# Set the locale
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get install --no-install-recommends -y locales
|
|
||||||
RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
|
|
||||||
RUN dpkg-reconfigure --frontend=noninteractive locales
|
|
||||||
RUN update-locale LANG=${LC_ALL}
|
|
||||||
|
|
||||||
# Instal apt-utils
|
|
||||||
RUN apt-get install --no-install-recommends -y \
|
|
||||||
apt-utils
|
|
||||||
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
RUN apt-get install --no-install-recommends -y \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
python \
|
|
||||||
libpq-dev \
|
|
||||||
linux-headers-generic \
|
|
||||||
gcc-10 \
|
|
||||||
make \
|
|
||||||
g++-10 \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
libtool \
|
|
||||||
patch \
|
|
||||||
libpcre3-dev \
|
|
||||||
libpcre3 \
|
|
||||||
python-dev \
|
|
||||||
libffi-dev \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev
|
|
||||||
|
|
||||||
# Use gcc 10
|
|
||||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Fetch and build CKAN and requirements
|
|
||||||
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
|
||||||
# Copy patches and apply patches script
|
|
||||||
COPY ./patches ${SRC_DIR}/patches
|
|
||||||
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
# Apply patches
|
|
||||||
RUN ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
RUN rm -rf /srv/app/src/ckan/.git
|
|
||||||
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
|
|
||||||
RUN pip wheel --wheel-dir=/wheels uwsgi==2.0.19.1 gevent==21.12.0 greenlet==1.1.3
|
|
||||||
|
|
||||||
|
|
||||||
###########################
|
|
||||||
### Default-Extensions ####
|
|
||||||
###########################
|
|
||||||
FROM ubuntu:focal-20210827 as extbuild
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
# List of default extensions
|
|
||||||
ENV DEFAULT_EXTENSIONS envvars
|
|
||||||
|
|
||||||
# Locations and tags, please use specific tags or revisions
|
|
||||||
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
|
|
||||||
ENV ENVVARS_GIT_BRANCH=0.0.1
|
|
||||||
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
python \
|
|
||||||
python-dev
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Fetch and build the default CKAN extensions
|
|
||||||
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
|
|
||||||
|
|
||||||
############
|
|
||||||
### MAIN ###
|
|
||||||
############
|
|
||||||
FROM ubuntu:focal-20210827
|
|
||||||
|
|
||||||
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
|
||||||
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
|
|
||||||
|
|
||||||
# Set timezone
|
|
||||||
ENV TZ=UTC
|
|
||||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
|
||||||
|
|
||||||
# Set Locale
|
|
||||||
ENV LC_ALL=en_US.UTF-8
|
|
||||||
|
|
||||||
# Set the locale
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y locales && \
|
|
||||||
sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \
|
|
||||||
dpkg-reconfigure --frontend=noninteractive locales && \
|
|
||||||
update-locale LANG=${LC_ALL} && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_DIR=${SRC_DIR}/ckan
|
|
||||||
ENV DATA_DIR=/srv/app/data
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y \
|
|
||||||
gettext \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
git \
|
|
||||||
libpq5 \
|
|
||||||
postgresql-client \
|
|
||||||
python \
|
|
||||||
libpython2.7 \
|
|
||||||
libmagic1 \
|
|
||||||
libpcre3 \
|
|
||||||
libxslt1.1 \
|
|
||||||
libxml2 \
|
|
||||||
tzdata \
|
|
||||||
apache2-utils && \
|
|
||||||
rm -rf /var/lib/apt/lists/* && \
|
|
||||||
# Create SRC_DIR
|
|
||||||
mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
|
||||||
|
|
||||||
# Get artifacts from build stages
|
|
||||||
COPY --from=ckanbuild /wheels /srv/app/wheels
|
|
||||||
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
|
||||||
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Additional install steps for build stages artifacts
|
|
||||||
RUN pip install --no-index --find-links=/srv/app/wheels uwsgi==2.0.19.1 gevent==21.12.0
|
|
||||||
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN groupadd -g 92 ckan && \
|
|
||||||
useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan
|
|
||||||
|
|
||||||
WORKDIR ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Install CKAN
|
|
||||||
RUN pip install -e /srv/app/src/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
|
|
||||||
# Install default CKAN extensions
|
|
||||||
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
|
|
||||||
# Create and update CKAN config
|
|
||||||
# Generate CKAN config
|
|
||||||
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
# Set the default level for extensions to INFO
|
|
||||||
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
|
|
||||||
# Create the data directory
|
|
||||||
mkdir ${DATA_DIR} && \
|
|
||||||
# Change ownership to app user
|
|
||||||
chown -R ckan:ckan /srv/app
|
|
||||||
|
|
||||||
# Remove wheels
|
|
||||||
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Copy necessary scripts
|
|
||||||
COPY setup/app ${APP_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Copy the alias script for paster to be ckan so it's compatible with 2.9
|
|
||||||
COPY setup/bin/ckan /usr/bin/ckan
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-entrypoint.d
|
|
||||||
|
|
||||||
# Create afterinit directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-afterinit.d
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
shopt -s nullglob
|
|
||||||
for patch in patches/*.patch; do
|
|
||||||
/usr/bin/patch -p0 -i $patch
|
|
||||||
done
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# this is called before uwsgi is executed
|
|
||||||
# uset his to add extra scripts before ckan is started
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
<!--
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Maintenance</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Maintenance</h1>
|
|
||||||
<p>Our data portal is currently in maintenance, please try in a while.</p>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
<!--
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Maintenance</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Maintenance</h1>
|
|
||||||
<p>Our data portal is currently in maintenance, please try in a while.</p>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
"""
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from SimpleHTTPServer import SimpleHTTPRequestHandler
|
|
||||||
from BaseHTTPServer import HTTPServer
|
|
||||||
from SocketServer import ThreadingMixIn
|
|
||||||
import os
|
|
||||||
|
|
||||||
PORT = 5000
|
|
||||||
|
|
||||||
web_dir = os.path.join(os.path.dirname(__file__))
|
|
||||||
os.chdir(web_dir)
|
|
||||||
|
|
||||||
Handler = SimpleHTTPRequestHandler
|
|
||||||
|
|
||||||
|
|
||||||
class MaintenanceServer(ThreadingMixIn, HTTPServer):
|
|
||||||
"""Handle requests in a separate thread."""
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
httpd = MaintenanceServer(("0.0.0.0", PORT), Handler)
|
|
||||||
print("Starting maintenance mode")
|
|
||||||
httpd.serve_forever()
|
|
||||||
|
|
||||||
|
|
@ -1,200 +0,0 @@
|
||||||
"""
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import subprocess
|
|
||||||
import psycopg2
|
|
||||||
import urllib2
|
|
||||||
import re
|
|
||||||
|
|
||||||
import time
|
|
||||||
|
|
||||||
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
|
|
||||||
|
|
||||||
RETRY = 5
|
|
||||||
|
|
||||||
def check_db_connection(retry=None):
|
|
||||||
|
|
||||||
print('[prerun] Start check_db_connection...')
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
|
|
||||||
try:
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print((str(e)))
|
|
||||||
print('[prerun] Unable to connect to the database...try again in a while.')
|
|
||||||
import time
|
|
||||||
time.sleep(10)
|
|
||||||
check_db_connection(retry = retry - 1)
|
|
||||||
else:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
def check_solr_connection(retry=None):
|
|
||||||
|
|
||||||
print('[prerun] Start check_solr_connection...')
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
url = os.environ.get('CKAN_SOLR_URL', '')
|
|
||||||
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection = urllib2.urlopen(search_url)
|
|
||||||
except urllib2.URLError as e:
|
|
||||||
print('[prerun] Unable to connect to solr...try again in a while.')
|
|
||||||
import time
|
|
||||||
time.sleep(10)
|
|
||||||
check_solr_connection(retry = retry - 1)
|
|
||||||
else:
|
|
||||||
import re
|
|
||||||
conn_info = connection.read()
|
|
||||||
# SolrCloud
|
|
||||||
conn_info = re.sub(r'"zkConnected":true', '"zkConnected":True', conn_info)
|
|
||||||
eval(conn_info)
|
|
||||||
|
|
||||||
def init_db():
|
|
||||||
|
|
||||||
print('[prerun] Start init_db...')
|
|
||||||
|
|
||||||
db_command = ['paster', '--plugin=ckan', 'db', 'init', '-c', ckan_ini]
|
|
||||||
|
|
||||||
print('[prerun] Initializing or upgrading db - start using paster db init')
|
|
||||||
try:
|
|
||||||
# run init scripts
|
|
||||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
|
||||||
|
|
||||||
print('[prerun] Initializing or upgrading db - end')
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
import time
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print((e.output))
|
|
||||||
raise e
|
|
||||||
print('[prerun] Initializing or upgrading db - finish')
|
|
||||||
|
|
||||||
|
|
||||||
def init_datastore():
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] Skipping datastore initialization')
|
|
||||||
return
|
|
||||||
|
|
||||||
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
|
|
||||||
'set-permissions', '-c', ckan_ini]
|
|
||||||
|
|
||||||
connection = psycopg2.connect(conn_str)
|
|
||||||
cursor = connection.cursor()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - start')
|
|
||||||
try:
|
|
||||||
datastore_perms = subprocess.Popen(
|
|
||||||
datastore_perms_command,
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
perms_sql = datastore_perms.stdout.read()
|
|
||||||
# Remove internal pg command as psycopg2 does not like it
|
|
||||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql.decode('utf-8'))
|
|
||||||
cursor.execute(perms_sql)
|
|
||||||
for notice in connection.notices:
|
|
||||||
print(notice)
|
|
||||||
|
|
||||||
connection.commit()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - end')
|
|
||||||
print((datastore_perms.stdout.read()))
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print('[prerun] Could not initialize datastore')
|
|
||||||
print((str(e)))
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in e.output:
|
|
||||||
print((e.output))
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print((e.output))
|
|
||||||
raise e
|
|
||||||
finally:
|
|
||||||
cursor.close()
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def create_sysadmin():
|
|
||||||
|
|
||||||
print('[prerun] Start create_sysadmin...')
|
|
||||||
|
|
||||||
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
|
||||||
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
|
||||||
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
|
||||||
|
|
||||||
if name and password and email:
|
|
||||||
|
|
||||||
# Check if user exists
|
|
||||||
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
|
|
||||||
|
|
||||||
out = subprocess.check_output(command)
|
|
||||||
if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')):
|
|
||||||
print('[prerun] Sysadmin user exists, skipping creation')
|
|
||||||
return
|
|
||||||
|
|
||||||
# Create user
|
|
||||||
command = ['paster', '--plugin=ckan', 'user', 'add',
|
|
||||||
name,
|
|
||||||
'password=' + password,
|
|
||||||
'email=' + email,
|
|
||||||
'-c', ckan_ini]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print(('[prerun] Created user {0}'.format(name)))
|
|
||||||
|
|
||||||
# Make it sysadmin
|
|
||||||
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
|
|
||||||
name,
|
|
||||||
'-c', ckan_ini]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print(('[prerun] Made user {0} a sysadmin'.format(name)))
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
|
||||||
|
|
||||||
if maintenance:
|
|
||||||
print('[prerun] Maintenance mode, skipping setup...')
|
|
||||||
else:
|
|
||||||
check_db_connection()
|
|
||||||
check_solr_connection()
|
|
||||||
init_db()
|
|
||||||
if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
|
|
||||||
init_datastore()
|
|
||||||
create_sysadmin()
|
|
||||||
|
|
@ -1,74 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
# Run any startup scripts provided by images extending this one
|
|
||||||
if [[ -d "${APP_DIR}/docker-entrypoint.d" ]]
|
|
||||||
then
|
|
||||||
for f in ${APP_DIR}/docker-entrypoint.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add session secret from chart
|
|
||||||
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET ]];then
|
|
||||||
echo "Not all environment variables are set. Generating sessions..."
|
|
||||||
else
|
|
||||||
echo "Setting session secrets from environment variables"
|
|
||||||
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
|
|
||||||
then
|
|
||||||
echo "Setting beaker.session.secret in ini file"
|
|
||||||
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python -c 'import secrets; print(secrets.token_urlsafe())')"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Set the common uwsgi options
|
|
||||||
echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
|
|
||||||
|
|
||||||
UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger /srv/app/production.ini --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch"
|
|
||||||
|
|
||||||
# Run the prerun script to init CKAN and create the default admin user
|
|
||||||
python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }
|
|
||||||
|
|
||||||
# Check if we are in maintenance mode and if yes serve the maintenance pages
|
|
||||||
if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi
|
|
||||||
|
|
||||||
# Run any after prerun/init scripts provided by images extending this one
|
|
||||||
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
|
|
||||||
then
|
|
||||||
for f in ${APP_DIR}/docker-afterinit.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
|
||||||
if [ $? -eq 0 ]
|
|
||||||
then
|
|
||||||
if [ "$PASSWORD_PROTECT" = true ]
|
|
||||||
then
|
|
||||||
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
|
||||||
then
|
|
||||||
# Generate htpasswd file for basicauth
|
|
||||||
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
|
||||||
# Start uwsgi with basicauth
|
|
||||||
uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
|
|
||||||
else
|
|
||||||
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
# Start uwsgi
|
|
||||||
uwsgi $UWSGI_OPTS
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "[prerun] failed...not starting CKAN."
|
|
||||||
fi
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
# CKAN <2.9 uses paster and the ckan plugin for paster which expects the configuration to be passed
|
|
||||||
# to a subcommand, example: paster --plugin=ckan datastore -c /srv/app/production.ini set-permissions
|
|
||||||
# CKAN >2.9 ckan CLI command expects the configuration option as the second and third parameter
|
|
||||||
# Example: ckan -c /srv/app/production.ini datastore set-permissions
|
|
||||||
#
|
|
||||||
# Shift three arguments so that we can reorder the argument list
|
|
||||||
# ckan -c /srv/app/production.ini datastore set-permissions
|
|
||||||
# becomes
|
|
||||||
# paster --plugin=ckan datastore -c /srv/app/production.ini set-permissions
|
|
||||||
config_switch=$1
|
|
||||||
config_file=$2
|
|
||||||
subcommand=$3
|
|
||||||
shift 3
|
|
||||||
paster --plugin=ckan "$subcommand" "$config_switch" "$config_file" "$@"
|
|
||||||
|
|
@ -1,203 +0,0 @@
|
||||||
##################
|
|
||||||
### Build CKAN ###
|
|
||||||
##################
|
|
||||||
FROM alpine:3.13.7 as ckanbuild
|
|
||||||
|
|
||||||
# Used by Github Actions to tag the image with
|
|
||||||
ENV IMAGE_TAG=2.9.11
|
|
||||||
|
|
||||||
# Set CKAN version to build
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
ENV GIT_BRANCH=ckan-2.9.11
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${SRC_DIR}
|
|
||||||
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
python3 \
|
|
||||||
python3-dev \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
postgresql-dev \
|
|
||||||
linux-headers \
|
|
||||||
gcc \
|
|
||||||
make \
|
|
||||||
g++ \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
libtool \
|
|
||||||
patch \
|
|
||||||
musl-dev \
|
|
||||||
pcre-dev \
|
|
||||||
pcre \
|
|
||||||
libffi-dev \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev
|
|
||||||
|
|
||||||
# Link python to python3
|
|
||||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
# Downgrade setuptools so that CKAN requirements can be built
|
|
||||||
RUN pip install setuptools==44.1.0
|
|
||||||
|
|
||||||
# Fetch and build CKAN and requirements
|
|
||||||
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
|
||||||
# Copy patches and apply patches script
|
|
||||||
COPY ./patches ${SRC_DIR}/patches
|
|
||||||
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
# Apply patches
|
|
||||||
RUN cd ${SRC_DIR} && ls -lah ${SRC_DIR} && ash ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
RUN rm -rf /srv/app/src/ckan/.git
|
|
||||||
|
|
||||||
# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736
|
|
||||||
RUN echo 'Cython < 3.0' > /tmp/constraint.txt
|
|
||||||
RUN PIP_CONSTRAINT=/tmp/constraint.txt pip wheel --wheel-dir=/wheels PyYAML==5.4.1
|
|
||||||
|
|
||||||
# RUN pip-compile ckan/requirements.in
|
|
||||||
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
|
|
||||||
RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3
|
|
||||||
|
|
||||||
###########################
|
|
||||||
### Default-Extensions ####
|
|
||||||
###########################
|
|
||||||
FROM alpine:3.13.7 as extbuild
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
# List of default extensions
|
|
||||||
ENV DEFAULT_EXTENSIONS envvars
|
|
||||||
|
|
||||||
# Locations and tags, please use specific tags or revisions
|
|
||||||
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
|
|
||||||
ENV ENVVARS_GIT_BRANCH=0.0.1
|
|
||||||
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
python3 \
|
|
||||||
python3-dev \
|
|
||||||
git \
|
|
||||||
curl
|
|
||||||
|
|
||||||
# Link python to python3
|
|
||||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
# Fetch and build the default CKAN extensions
|
|
||||||
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
|
|
||||||
|
|
||||||
############
|
|
||||||
### MAIN ###
|
|
||||||
############
|
|
||||||
FROM alpine:3.13.7
|
|
||||||
|
|
||||||
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
|
||||||
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_DIR=${SRC_DIR}/ckan
|
|
||||||
ENV DATA_DIR=/srv/app/data
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
RUN apk add --no-cache \
|
|
||||||
python3 \
|
|
||||||
bash \
|
|
||||||
git \
|
|
||||||
gettext \
|
|
||||||
curl \
|
|
||||||
postgresql-client \
|
|
||||||
libmagic \
|
|
||||||
pcre \
|
|
||||||
libxslt \
|
|
||||||
libxml2 \
|
|
||||||
tzdata \
|
|
||||||
apache2-utils && \
|
|
||||||
# Create SRC_DIR
|
|
||||||
mkdir -p ${SRC_DIR} && \
|
|
||||||
# Link python to python3
|
|
||||||
ln -s /usr/bin/python3 /usr/bin/python
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
# Get artifacts from build stages
|
|
||||||
COPY --from=ckanbuild /wheels /srv/app/wheels
|
|
||||||
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
|
||||||
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Additional install steps for build stages artifacts
|
|
||||||
RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0
|
|
||||||
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN addgroup -g 92 -S ckan && \
|
|
||||||
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
|
||||||
|
|
||||||
WORKDIR ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Install CKAN
|
|
||||||
RUN pip install -e /srv/app/src/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
|
|
||||||
# Install default CKAN extensions
|
|
||||||
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
|
|
||||||
# Create and update CKAN config
|
|
||||||
# Set timezone
|
|
||||||
echo "UTC" > /etc/timezone && \
|
|
||||||
# Generate CKAN config
|
|
||||||
ckan generate config ${APP_DIR}/production.ini && \
|
|
||||||
ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
|
|
||||||
# Configure plugins
|
|
||||||
ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
# Create the data directory
|
|
||||||
mkdir ${DATA_DIR} && \
|
|
||||||
# Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created
|
|
||||||
ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \
|
|
||||||
# Set the default level for extensions to INFO
|
|
||||||
ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
|
|
||||||
# Change ownership to app user
|
|
||||||
chown -R ckan:ckan /srv/app
|
|
||||||
|
|
||||||
RUN pip install sqlalchemy==1.3.19
|
|
||||||
# Remove wheels
|
|
||||||
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Copy necessary scripts
|
|
||||||
COPY setup/app ${APP_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-entrypoint.d
|
|
||||||
|
|
||||||
# Create afterinit directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-afterinit.d
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,244 +0,0 @@
|
||||||
##################
|
|
||||||
### Build CKAN ###
|
|
||||||
##################
|
|
||||||
FROM ubuntu:focal-20210827 as ckanbuild
|
|
||||||
|
|
||||||
# Used by Github Actions to tag the image with
|
|
||||||
ENV IMAGE_TAG=2.9.11-focal
|
|
||||||
|
|
||||||
# Set CKAN version to build
|
|
||||||
ENV GIT_URL=https://github.com/ckan/ckan.git
|
|
||||||
ENV GIT_BRANCH=ckan-2.9.11
|
|
||||||
|
|
||||||
# Set timezone
|
|
||||||
ENV TZ=UTC
|
|
||||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
|
||||||
|
|
||||||
# Set Locale
|
|
||||||
ENV LC_ALL=en_US.UTF-8
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${SRC_DIR}
|
|
||||||
|
|
||||||
# Set the locale
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get install --no-install-recommends -y locales
|
|
||||||
RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
|
|
||||||
RUN dpkg-reconfigure --frontend=noninteractive locales
|
|
||||||
RUN update-locale LANG=${LC_ALL}
|
|
||||||
|
|
||||||
# Instal apt-utils
|
|
||||||
RUN apt-get install --no-install-recommends -y \
|
|
||||||
apt-utils
|
|
||||||
|
|
||||||
# Packages to build CKAN requirements and plugins
|
|
||||||
RUN apt-get install --no-install-recommends -y \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
python3 \
|
|
||||||
libpq-dev \
|
|
||||||
linux-headers-generic \
|
|
||||||
gcc-10 \
|
|
||||||
make \
|
|
||||||
g++-10 \
|
|
||||||
autoconf \
|
|
||||||
automake \
|
|
||||||
libtool \
|
|
||||||
patch \
|
|
||||||
libpcre3-dev \
|
|
||||||
libpcre3 \
|
|
||||||
python3-dev \
|
|
||||||
libffi-dev \
|
|
||||||
libxml2-dev \
|
|
||||||
libxslt-dev
|
|
||||||
|
|
||||||
# Use gcc 10
|
|
||||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10
|
|
||||||
|
|
||||||
# Link python to python3
|
|
||||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
# Downgrade setuptools so that CKAN requirements can be built
|
|
||||||
RUN pip install setuptools==44.1.0
|
|
||||||
|
|
||||||
# Fetch and build CKAN and requirements
|
|
||||||
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
|
||||||
# Copy patches and apply patches script
|
|
||||||
COPY ./patches ${SRC_DIR}/patches
|
|
||||||
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
# Apply patches
|
|
||||||
RUN ${SRC_DIR}/apply_ckan_patches.sh
|
|
||||||
RUN rm -rf /srv/app/src/ckan/.git
|
|
||||||
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
|
|
||||||
RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3
|
|
||||||
|
|
||||||
|
|
||||||
###########################
|
|
||||||
### Default-Extensions ####
|
|
||||||
###########################
|
|
||||||
FROM ubuntu:focal-20210827 as extbuild
|
|
||||||
|
|
||||||
# Set src dirs
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
|
|
||||||
# List of default extensions
|
|
||||||
ENV DEFAULT_EXTENSIONS envvars
|
|
||||||
|
|
||||||
# Locations and tags, please use specific tags or revisions
|
|
||||||
ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
|
|
||||||
ENV ENVVARS_GIT_BRANCH=0.0.1
|
|
||||||
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y \
|
|
||||||
git \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
python3 \
|
|
||||||
python3-dev
|
|
||||||
|
|
||||||
# Link python to python3
|
|
||||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
|
||||||
|
|
||||||
# Create the src directory
|
|
||||||
RUN mkdir -p ${SRC_DIR}
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
# Downgrade setuptools so that CKAN requirements can be built
|
|
||||||
RUN pip install setuptools==44.1.0
|
|
||||||
|
|
||||||
# Fetch and build the default CKAN extensions
|
|
||||||
RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
|
|
||||||
|
|
||||||
############
|
|
||||||
### MAIN ###
|
|
||||||
############
|
|
||||||
FROM ubuntu:focal-20210827
|
|
||||||
|
|
||||||
LABEL maintainer="Keitaro Inc <info@keitaro.com>"
|
|
||||||
LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
|
|
||||||
|
|
||||||
# Set timezone
|
|
||||||
ENV TZ=UTC
|
|
||||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
|
||||||
|
|
||||||
# Set Locale
|
|
||||||
ENV LC_ALL=en_US.UTF-8
|
|
||||||
|
|
||||||
# Set the locale
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y locales && \
|
|
||||||
sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \
|
|
||||||
dpkg-reconfigure --frontend=noninteractive locales && \
|
|
||||||
update-locale LANG=${LC_ALL} && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
ENV APP_DIR=/srv/app
|
|
||||||
ENV SRC_DIR=/srv/app/src
|
|
||||||
ENV CKAN_DIR=${SRC_DIR}/ckan
|
|
||||||
ENV DATA_DIR=/srv/app/data
|
|
||||||
ENV PIP_SRC=${SRC_DIR}
|
|
||||||
ENV CKAN_SITE_URL=http://localhost:5000
|
|
||||||
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
|
|
||||||
|
|
||||||
# Install necessary packages to run CKAN
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --no-install-recommends -y \
|
|
||||||
gettext \
|
|
||||||
curl \
|
|
||||||
ca-certificates \
|
|
||||||
libpq5 \
|
|
||||||
git \
|
|
||||||
postgresql-client \
|
|
||||||
python3 \
|
|
||||||
python3-distutils \
|
|
||||||
libpython3.8 \
|
|
||||||
libmagic1 \
|
|
||||||
libpcre3 \
|
|
||||||
libxslt1.1 \
|
|
||||||
libxml2 \
|
|
||||||
tzdata \
|
|
||||||
apache2-utils && \
|
|
||||||
rm -rf /var/lib/apt/lists/* && \
|
|
||||||
# Create SRC_DIR
|
|
||||||
mkdir -p ${SRC_DIR} && \
|
|
||||||
# Link python to python3
|
|
||||||
ln -s /usr/bin/python3 /usr/bin/python
|
|
||||||
|
|
||||||
# Install pip
|
|
||||||
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
|
||||||
python ${SRC_DIR}/get-pip.py
|
|
||||||
|
|
||||||
# Downgrade setuptools so that CKAN requirements can be built
|
|
||||||
RUN pip install setuptools==44.1.0
|
|
||||||
|
|
||||||
# Get artifacts from build stages
|
|
||||||
COPY --from=ckanbuild /wheels /srv/app/wheels
|
|
||||||
COPY --from=extbuild /wheels /srv/app/ext_wheels
|
|
||||||
COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Additional install steps for build stages artifacts
|
|
||||||
RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0
|
|
||||||
|
|
||||||
# Create a local user and group to run the app
|
|
||||||
RUN groupadd -g 92 ckan && \
|
|
||||||
useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan
|
|
||||||
|
|
||||||
WORKDIR ${CKAN_DIR}
|
|
||||||
|
|
||||||
# Install CKAN
|
|
||||||
RUN pip install -e /srv/app/src/ckan && \
|
|
||||||
cp who.ini ${APP_DIR} && \
|
|
||||||
pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
|
|
||||||
# Install default CKAN extensions
|
|
||||||
pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
|
|
||||||
# Create and update CKAN config
|
|
||||||
# Generate CKAN config
|
|
||||||
ckan generate config ${APP_DIR}/production.ini && \
|
|
||||||
# Configure plugins
|
|
||||||
ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
|
|
||||||
ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
|
|
||||||
# Create the data directory
|
|
||||||
mkdir ${DATA_DIR} && \
|
|
||||||
# Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created
|
|
||||||
ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \
|
|
||||||
# Set the default level for extensions to INFO
|
|
||||||
ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
|
|
||||||
# Change ownership to app user
|
|
||||||
chown -R ckan:ckan /srv/app
|
|
||||||
|
|
||||||
# Remove wheels
|
|
||||||
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
|
|
||||||
|
|
||||||
# Copy necessary scripts
|
|
||||||
COPY setup/app ${APP_DIR}
|
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
|
||||||
|
|
||||||
# Create entrypoint directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-entrypoint.d
|
|
||||||
|
|
||||||
# Create afterinit directory for children image scripts
|
|
||||||
ONBUILD RUN mkdir docker-afterinit.d
|
|
||||||
|
|
||||||
EXPOSE 5000
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
|
||||||
|
|
||||||
USER ckan
|
|
||||||
|
|
||||||
CMD ["/srv/app/start_ckan.sh"]
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
--- ckan/ckan/model/__init__.py
|
|
||||||
+++ ckan/ckan/model/__init__.py
|
|
||||||
@@ -276,7 +276,7 @@ class Repository():
|
|
||||||
self.reset_alembic_output()
|
|
||||||
alembic_config = AlembicConfig(self._alembic_ini)
|
|
||||||
alembic_config.set_main_option(
|
|
||||||
- "sqlalchemy.url", config.get("sqlalchemy.url")
|
|
||||||
+ "sqlalchemy.url", config.get("sqlalchemy.url").replace('%', '%%')
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
sqlalchemy_migrate_version = self.metadata.bind.execute(
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
--- ckan/ckanext/datastore/backend/postgres.py
|
|
||||||
+++ ckan/ckanext/datastore/backend/postgres.py
|
|
||||||
@@ -1809,7 +1809,7 @@ class DatastorePostgresqlBackend(DatastoreBackend):
|
|
||||||
read only user.
|
|
||||||
'''
|
|
||||||
write_connection = self._get_write_engine().connect()
|
|
||||||
- read_connection_user = sa_url.make_url(self.read_url).username
|
|
||||||
+ read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0]
|
|
||||||
|
|
||||||
drop_foo_sql = u'DROP TABLE IF EXISTS _foo'
|
|
||||||
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
shopt -s nullglob
|
|
||||||
for patch in patches/*.patch; do
|
|
||||||
/usr/bin/patch -p0 -i $patch
|
|
||||||
done
|
|
||||||
|
|
@ -1,39 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
|
||||||
|
|
||||||
socket = /tmp/uwsgi.sock
|
|
||||||
uid = ckan
|
|
||||||
gid = ckan
|
|
||||||
http = :5000
|
|
||||||
|
|
||||||
master = true
|
|
||||||
enable-threads = true
|
|
||||||
lazy-apps = true
|
|
||||||
gevent-early-monkey-patch = true
|
|
||||||
vacuum = true
|
|
||||||
single-interpreter= true
|
|
||||||
die-on-term = true
|
|
||||||
need-app = true
|
|
||||||
auto-procname = true
|
|
||||||
|
|
||||||
wsgi-file = /srv/app/wsgi.py
|
|
||||||
module = wsgi:application
|
|
||||||
gevent = 2000
|
|
||||||
callable = application
|
|
||||||
paste = config:/srv/app/production.ini
|
|
||||||
paste-logger = /srv/app/production.ini
|
|
||||||
|
|
||||||
post-buffering = 1
|
|
||||||
buffer-size= 12288
|
|
||||||
max-requests = 3000
|
|
||||||
max-worker-lifetime = 3600
|
|
||||||
reload-on-rss = 4096
|
|
||||||
worker-reload-mercy = 60
|
|
||||||
socket-timeout = 300
|
|
||||||
queue = 1000
|
|
||||||
queue-blocksize = 204800
|
|
||||||
static-gzip-all = true
|
|
||||||
listen = 1000
|
|
||||||
http-timeout = 1000
|
|
||||||
http-headers-timeout = 1000
|
|
||||||
http-connect-timeout = 1000
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# this is called before uwsgi is executed
|
|
||||||
# uset his to add extra scripts before ckan is started
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
<!--
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Maintenance</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Maintenance</h1>
|
|
||||||
<p>Our data portal is currently in maintenance, please try in a while.</p>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
<!--
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Maintenance</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Maintenance</h1>
|
|
||||||
<p>Our data portal is currently in maintenance, please try in a while.</p>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
"""
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
|
|
||||||
import os
|
|
||||||
|
|
||||||
PORT = 5000
|
|
||||||
|
|
||||||
web_dir = os.path.join(os.path.dirname(__file__))
|
|
||||||
os.chdir(web_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def run(server_class=ThreadingHTTPServer, handler_class=SimpleHTTPRequestHandler):
|
|
||||||
server_address = ("0.0.0.0", PORT)
|
|
||||||
httpd = server_class(server_address, handler_class)
|
|
||||||
print("Starting maintenance mode")
|
|
||||||
httpd.serve_forever()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
run()
|
|
||||||
|
|
||||||
|
|
@ -1,228 +0,0 @@
|
||||||
"""
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from multiprocessing import connection
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import subprocess
|
|
||||||
import psycopg2
|
|
||||||
from sqlalchemy.engine.url import make_url
|
|
||||||
import urllib.request, urllib.error, urllib.parse, base64
|
|
||||||
import re
|
|
||||||
import json
|
|
||||||
|
|
||||||
import time
|
|
||||||
|
|
||||||
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
|
|
||||||
|
|
||||||
RETRY = 5
|
|
||||||
|
|
||||||
def check_db_connection(retry=None):
|
|
||||||
|
|
||||||
print('[prerun] Start check_db_connection...')
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
|
|
||||||
try:
|
|
||||||
db_user = make_url(conn_str).username
|
|
||||||
db_passwd = make_url(conn_str).password
|
|
||||||
db_host = make_url(conn_str).host
|
|
||||||
db_name = make_url(conn_str).database
|
|
||||||
connection = psycopg2.connect(user=db_user,
|
|
||||||
host=db_host,
|
|
||||||
password=db_passwd,
|
|
||||||
database=db_name)
|
|
||||||
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print((str(e)))
|
|
||||||
print('[prerun] Unable to connect to the database...try again in a while.')
|
|
||||||
import time
|
|
||||||
time.sleep(10)
|
|
||||||
check_db_connection(retry = retry - 1)
|
|
||||||
else:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
def check_solr_connection(retry=None):
|
|
||||||
|
|
||||||
print('[prerun] Start check_solr_connection...')
|
|
||||||
|
|
||||||
if retry is None:
|
|
||||||
retry = RETRY
|
|
||||||
elif retry == 0:
|
|
||||||
print('[prerun] Giving up after 5 tries...')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
url = os.environ.get('CKAN_SOLR_URL', '')
|
|
||||||
username = os.environ.get('CKAN_SOLR_USER', '')
|
|
||||||
password = os.environ.get('CKAN_SOLR_PASSWORD', '')
|
|
||||||
search_url = '{url}/schema/name?wt=json'.format(url=url)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
try:
|
|
||||||
if not username:
|
|
||||||
connection = urllib.request.urlopen(search_url)
|
|
||||||
else:
|
|
||||||
request = urllib.request.Request(search_url)
|
|
||||||
base64string = base64.b64encode(bytes('%s:%s' % (username, password),'ascii'))
|
|
||||||
request.add_header("Authorization", "Basic %s" % base64string.decode('utf-8'))
|
|
||||||
connection = urllib.request.urlopen(request)
|
|
||||||
except urllib.error.URLError as e:
|
|
||||||
print('[prerun] Unable to connect to solr...try again in a while.')
|
|
||||||
import time
|
|
||||||
time.sleep(10)
|
|
||||||
check_solr_connection(retry = retry - 1)
|
|
||||||
else:
|
|
||||||
import re
|
|
||||||
conn_info = connection.read()
|
|
||||||
schema_name = json.loads(conn_info)
|
|
||||||
if 'ckan' in schema_name['name']:
|
|
||||||
print('[prerun] Succesfully connected to solr and CKAN schema loaded')
|
|
||||||
else:
|
|
||||||
print('[prerun] Succesfully connected to solr, but CKAN schema not found')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def init_db():
|
|
||||||
|
|
||||||
print('[prerun] Start init_db...')
|
|
||||||
|
|
||||||
db_command = ['ckan', '-c', ckan_ini, 'db', 'init']
|
|
||||||
|
|
||||||
print('[prerun] Initializing or upgrading db - start using ckan db init')
|
|
||||||
try:
|
|
||||||
# run init scripts
|
|
||||||
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
|
||||||
|
|
||||||
print('[prerun] Initializing or upgrading db - end')
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in str(e.output):
|
|
||||||
print(e.output.decode('utf-8'))
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
import time
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print(e.output.decode('utf-8'))
|
|
||||||
raise e
|
|
||||||
print('[prerun] Initializing or upgrading db - finish')
|
|
||||||
|
|
||||||
|
|
||||||
def init_datastore():
|
|
||||||
|
|
||||||
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
|
||||||
if not conn_str:
|
|
||||||
print('[prerun] Skipping datastore initialization')
|
|
||||||
return
|
|
||||||
|
|
||||||
datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore',
|
|
||||||
'set-permissions']
|
|
||||||
|
|
||||||
db_user = make_url(conn_str).username
|
|
||||||
db_passwd = make_url(conn_str).password
|
|
||||||
db_host = make_url(conn_str).host
|
|
||||||
db_name = make_url(conn_str).database
|
|
||||||
connection = psycopg2.connect(user=db_user,
|
|
||||||
host=db_host,
|
|
||||||
password=db_passwd,
|
|
||||||
database=db_name)
|
|
||||||
cursor = connection.cursor()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - start')
|
|
||||||
try:
|
|
||||||
datastore_perms = subprocess.Popen(
|
|
||||||
datastore_perms_command,
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
perms_sql = datastore_perms.stdout.read()
|
|
||||||
perms_sql = perms_sql.decode('utf-8')
|
|
||||||
perms_sql = perms_sql.replace("@"+db_host, "")
|
|
||||||
# Remove internal pg command as psycopg2 does not like it
|
|
||||||
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
|
||||||
cursor.execute(perms_sql)
|
|
||||||
for notice in connection.notices:
|
|
||||||
print(notice)
|
|
||||||
|
|
||||||
connection.commit()
|
|
||||||
|
|
||||||
print('[prerun] Initializing datastore db - end')
|
|
||||||
print((datastore_perms.stdout.read()))
|
|
||||||
except psycopg2.Error as e:
|
|
||||||
print('[prerun] Could not initialize datastore')
|
|
||||||
print(e.decode('utf-8'))
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
if 'OperationalError' in str(e.output):
|
|
||||||
print(e.output.decode('utf-8'))
|
|
||||||
print('[prerun] Database not ready, waiting a bit before exit...')
|
|
||||||
time.sleep(5)
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
print(e.output.decode('utf-8'))
|
|
||||||
raise e
|
|
||||||
finally:
|
|
||||||
cursor.close()
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
|
|
||||||
def create_sysadmin():
|
|
||||||
|
|
||||||
print('[prerun] Start create_sysadmin...')
|
|
||||||
|
|
||||||
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
|
||||||
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
|
||||||
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
|
||||||
|
|
||||||
if name and password and email:
|
|
||||||
|
|
||||||
# Check if user exists
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'user', 'show', name]
|
|
||||||
|
|
||||||
out = subprocess.check_output(command)
|
|
||||||
if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')):
|
|
||||||
print('[prerun] Sysadmin user exists, skipping creation')
|
|
||||||
return
|
|
||||||
|
|
||||||
# Create user
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'user', 'add',
|
|
||||||
name,
|
|
||||||
'password=' + password,
|
|
||||||
'email=' + email]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print(('[prerun] Created user {0}'.format(name)))
|
|
||||||
|
|
||||||
# Make it sysadmin
|
|
||||||
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
|
|
||||||
name]
|
|
||||||
|
|
||||||
subprocess.call(command)
|
|
||||||
print(('[prerun] Made user {0} a sysadmin'.format(name)))
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
|
||||||
|
|
||||||
if maintenance:
|
|
||||||
print('[prerun] Maintenance mode, skipping setup...')
|
|
||||||
else:
|
|
||||||
check_db_connection()
|
|
||||||
check_solr_connection()
|
|
||||||
init_db()
|
|
||||||
if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
|
|
||||||
init_datastore()
|
|
||||||
create_sysadmin()
|
|
||||||
|
|
@ -1,76 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
# Run any startup scripts provided by images extending this one
|
|
||||||
if [[ -d "${APP_DIR}/docker-entrypoint.d" ]]
|
|
||||||
then
|
|
||||||
for f in ${APP_DIR}/docker-entrypoint.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add session secret from chart
|
|
||||||
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET || -z $JWT_ENCODE_SECRET || -v $JWT_ENCODE_SECRET || -z $JWT_DECODE_SECRET || -v $JWT_DECODE_SECRET ]];then
|
|
||||||
echo "Not all environment variables are set. Generating sessions..."
|
|
||||||
else
|
|
||||||
echo "Setting session secrets from environment variables"
|
|
||||||
ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
|
|
||||||
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_ENCODE_SECRET"
|
|
||||||
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_DECODE_SECRET"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
|
|
||||||
then
|
|
||||||
echo "Setting secrets in ini file"
|
|
||||||
ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
|
|
||||||
ckan config-tool $APP_DIR/production.ini "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
|
|
||||||
JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
|
|
||||||
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_SECRET"
|
|
||||||
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_SECRET"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Run the prerun script to init CKAN and create the default admin user
|
|
||||||
python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }
|
|
||||||
|
|
||||||
# Check if we are in maintenance mode and if yes serve the maintenance pages
|
|
||||||
if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi
|
|
||||||
|
|
||||||
# Run any after prerun/init scripts provided by images extending this one
|
|
||||||
if [[ -d "${APP_DIR}/docker-afterinit.d" ]]
|
|
||||||
then
|
|
||||||
for f in ${APP_DIR}/docker-afterinit.d/*; do
|
|
||||||
case "$f" in
|
|
||||||
*.sh) echo "$0: Running after prerun init file $f"; . "$f" ;;
|
|
||||||
*.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;;
|
|
||||||
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
|
||||||
esac
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
|
||||||
if [ $? -eq 0 ]
|
|
||||||
then
|
|
||||||
if [ "$PASSWORD_PROTECT" = true ]
|
|
||||||
then
|
|
||||||
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
|
||||||
then
|
|
||||||
# Generate htpasswd file for basicauth
|
|
||||||
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
|
||||||
# Start uwsgi with basicauth
|
|
||||||
uwsgi --ini /srv/app/basic-auth-uwsgi.conf -p ${UWSGI_PROC_NO:-2} --pcre-jit
|
|
||||||
else
|
|
||||||
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
# Start uwsgi
|
|
||||||
echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
|
|
||||||
uwsgi --ini /srv/app/uwsgi.conf -p ${UWSGI_PROC_NO:-2}
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "[prerun] failed...not starting CKAN."
|
|
||||||
fi
|
|
||||||
|
|
@ -1,37 +0,0 @@
|
||||||
[uwsgi]
|
|
||||||
socket = /tmp/uwsgi.sock
|
|
||||||
uid = ckan
|
|
||||||
gid = ckan
|
|
||||||
http = :5000
|
|
||||||
|
|
||||||
master = true
|
|
||||||
enable-threads = true
|
|
||||||
lazy-apps = true
|
|
||||||
gevent-early-monkey-patch = true
|
|
||||||
vacuum = true
|
|
||||||
single-interpreter= true
|
|
||||||
die-on-term = true
|
|
||||||
need-app = true
|
|
||||||
auto-procname = true
|
|
||||||
|
|
||||||
wsgi-file = /srv/app/wsgi.py
|
|
||||||
module = wsgi:application
|
|
||||||
gevent = 2000
|
|
||||||
callable = application
|
|
||||||
paste = config:/srv/app/production.ini
|
|
||||||
paste-logger = /srv/app/production.ini
|
|
||||||
|
|
||||||
post-buffering = 1
|
|
||||||
buffer-size= 12288
|
|
||||||
max-requests = 3000
|
|
||||||
max-worker-lifetime = 3600
|
|
||||||
reload-on-rss = 4096
|
|
||||||
worker-reload-mercy = 60
|
|
||||||
socket-timeout = 300
|
|
||||||
queue = 1000
|
|
||||||
queue-blocksize = 204800
|
|
||||||
static-gzip-all = true
|
|
||||||
listen = 1000
|
|
||||||
http-timeout = 1000
|
|
||||||
http-headers-timeout = 1000
|
|
||||||
http-connect-timeout = 1000
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
"""
|
|
||||||
Copyright (c) 2016 Keitaro AB
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
import os
|
|
||||||
from ckan.config.middleware import make_app
|
|
||||||
from ckan.cli import CKANConfigLoader
|
|
||||||
from logging.config import fileConfig as loggingFileConfig
|
|
||||||
config_filepath = os.path.join(
|
|
||||||
os.path.dirname(os.path.abspath(__file__)), u'production.ini')
|
|
||||||
abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
|
|
||||||
loggingFileConfig(config_filepath)
|
|
||||||
config = CKANConfigLoader(config_filepath).get_config()
|
|
||||||
application = make_app(config)
|
|
||||||
Loading…
Reference in New Issue