CatalogueRevamping
/
docker-ckan
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into ckan2.10

This commit is contained in:
Kiril-Poposki1998 2023-12-19 11:10:46 +01:00
parent efe53d5f56 a918f98418
commit be071e36a7
32 changed files with 936 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
.github/workflows/master_merge.yml vendored
View File

@ -311,3 +311,49 @@ jobs:
ghcr.io/keitaroinc/datapusher:${{ steps.datapusher.outputs.IMAGE_TAG }} ghcr.io/keitaroinc/datapusher:${{ steps.datapusher.outputs.IMAGE_TAG }}
cache-from: type=local,src=/tmp/.buildx-cache-datapusher cache-from: type=local,src=/tmp/.buildx-cache-datapusher
cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-datapusher cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-datapusher
build-psql-init:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.CR_PAT }}
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache-psql-init
key: ${{ runner.os }}-buildx-psql-init-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-psql-init
- name: Get docker tag for psql-init image
id: psql-init
run: |
echo "::set-output name=IMAGE_TAG::$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/psql-init/Dockerfile)"
- name: Build and push psql-init
uses: docker/build-push-action@v2
with:
context: ./images/psql-init
file: ./images/psql-init/Dockerfile
push: true
tags: |
keitaro/psql-init:${{ steps.psql-init.outputs.IMAGE_TAG }}
ghcr.io/keitaroinc/psql-init:${{ steps.psql-init.outputs.IMAGE_TAG }}
cache-from: type=local,src=/tmp/.buildx-cache-psql-init
cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-psql-init

31
.github/workflows/pr_checks.yml vendored
View File

@ -233,3 +233,34 @@ jobs:
tags: keitaro/ckandatapusher:${{ steps.datapusher.outputs.IMAGE_TAG }} tags: keitaro/ckandatapusher:${{ steps.datapusher.outputs.IMAGE_TAG }}
cache-from: type=local,src=/tmp/.buildx-cache-datapusher cache-from: type=local,src=/tmp/.buildx-cache-datapusher
cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-datapusher cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-datapusher
build-psql-init:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache-psql-init
key: ${{ runner.os }}-buildx-psql-init-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-psql-init
- name: Get docker tag for psql-init image
id: psql-init
run: |
echo "::set-output name=IMAGE_TAG::$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/psql-init/Dockerfile)"
- name: Build psql-init
uses: docker/build-push-action@v2
with:
context: ./images/psql-init
file: ./images/psql-init/Dockerfile
push: false
tags: keitaro/psql-init:${{ steps.psql-init.outputs.IMAGE_TAG }}
cache-from: type=local,src=/tmp/.buildx-cache-psql-init
cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-psql-init

8
Readme.md
View File

@ -48,7 +48,7 @@ We recommend to use a multi-stage approach to extend the docker images that we p
################### ###################
### Extensions #### ### Extensions ####
################### ###################
FROM ghcr.io/keitaroinc/ckan:2.9.7 as extbuild FROM ghcr.io/keitaroinc/ckan:2.9.9 as extbuild
# Switch to the root user # Switch to the root user
USER root USER root
@ -64,7 +64,7 @@ RUN pip wheel --wheel-dir=/wheels git+https://github.com/acmecorp/ckanext-acme@0
############ ############
### MAIN ### ### MAIN ###
############ ############
FROM ghcr.io/keitaroinc/ckan:2.9.7 FROM ghcr.io/keitaroinc/ckan:2.9.9
# Add the custom extensions to the plugins list # Add the custom extensions to the plugins list
ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher acme ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher acme
@ -94,9 +94,9 @@ You can add scripts to CKAN custom images and copy them to the *docker-afterinit
## Build ## Build
To build a CKAN image run: To build a CKAN image run:
```sh ```sh
docker build --tag ghcr.io/keitaroinc/ckan:2.9.7 images/ckan/2.9 docker build --tag ghcr.io/keitaroinc/ckan:2.9.9 images/ckan/2.9
``` ```
The –-tag ghcr.io/keitaroinc/ckan:2.9.7 flag sets the image name to ghcr.io/keitaroinc/ckan:2.9.7 and 'images/ckan/2.9' at the end tells docker build to use the context into the specified directory where the Dockerfile and related contents are. The –-tag ghcr.io/keitaroinc/ckan:2.9.9 flag sets the image name to ghcr.io/keitaroinc/ckan:2.9.9 and 'images/ckan/2.9' at the end tells docker build to use the context into the specified directory where the Dockerfile and related contents are.
## Upload to DockerHub ## Upload to DockerHub
>*It's recommended to upload built images to DockerHub* >*It's recommended to upload built images to DockerHub*

203
compose/2.8/solr/solrconfig-2.9.8.xml Normal file
View File

@ -0,0 +1,203 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
NB Please copy changes to this file into the multilingual schema:
ckanext/multilingual/solr/schema.xml
-->
<!-- We update the version when there is a backward-incompatible change to this
schema. We used to use the `version` attribute for this but this is an internal
attribute that should not be used so starting from CKAN 2.10 we use the `name`
attribute with the form `ckan-X.Y` -->
<schema name="ckan-2.10" version="1.6">
<types>
<fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
<fieldtype name="binary" class="solr.BinaryField"/>
<fieldType name="int" class="solr.IntPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="float" class="solr.FloatPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="long" class="solr.LongPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="double" class="solr.DoublePointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pint" class="solr.IntPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pfloat" class="solr.FloatPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="plong" class="solr.LongPointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pdouble" class="solr.DoublePointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="date" class="solr.DatePointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pdate" class="solr.DatePointField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="pdates" class="solr.DatePointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
<fieldType name="pints" class="solr.IntPointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="pfloats" class="solr.FloatPointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="plongs" class="solr.LongPointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="pdoubles" class="solr.DoublePointField" positionIncrementGap="0" multiValued="true"/>
<fieldType name="text" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
<filter class="solr.ASCIIFoldingFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
<filter class="solr.ASCIIFoldingFilterFactory"/>
</analyzer>
</fieldType>
<!-- A general unstemmed text field - good if one does not know the language of the field -->
<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
<filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<fieldType name="text_ngram" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.NGramTokenizerFactory" minGramSize="2" maxGramSize="10"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
</types>
<fields>
<field name="index_id" type="string" indexed="true" stored="true" required="true" />
<field name="id" type="string" indexed="true" stored="true" required="true" />
<field name="site_id" type="string" indexed="true" stored="true" required="true" />
<field name="title" type="text" indexed="true" stored="true" />
<field name="title_ngram" type="text_ngram" indexed="true" stored="true" />
<field name="entity_type" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="dataset_type" type="string" indexed="true" stored="true" />
<field name="state" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="name" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="name_ngram" type="text_ngram" indexed="true" stored="true" />
<field name="revision_id" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="version" type="string" indexed="true" stored="true" />
<field name="url" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="ckan_url" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="download_url" type="string" indexed="true" stored="true" omitNorms="true" />
<field name="notes" type="text" indexed="true" stored="true"/>
<field name="author" type="text_general" indexed="true" stored="true" />
<field name="author_email" type="text_general" indexed="true" stored="true" />
<field name="maintainer" type="text_general" indexed="true" stored="true" />
<field name="maintainer_email" type="text_general" indexed="true" stored="true" />
<field name="license" type="string" indexed="true" stored="true" />
<field name="license_id" type="string" indexed="true" stored="true" />
<field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="organization" type="string" indexed="true" stored="true" multiValued="false"/>
<field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
<field name="permission_labels" type="string" indexed="true" stored="false" multiValued="true"/>
<field name="res_name" type="text_general" indexed="true" stored="true" multiValued="true" />
<field name="res_description" type="text_general" indexed="true" stored="true" multiValued="true"/>
<field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="res_type" type="string" indexed="true" stored="true" multiValued="true"/>
<!-- catchall field, containing all other searchable text fields (implemented
via copyField further on in this schema -->
<field name="text" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="urls" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="depends_on" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="dependency_of" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="derives_from" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="has_derivation" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="links_to" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
<field name="views_total" type="int" indexed="true" stored="false"/>
<field name="views_recent" type="int" indexed="true" stored="false"/>
<field name="resources_accessed_total" type="int" indexed="true" stored="false"/>
<field name="resources_accessed_recent" type="int" indexed="true" stored="false"/>
<field name="metadata_created" type="date" indexed="true" stored="true" multiValued="false"/>
<field name="metadata_modified" type="date" indexed="true" stored="true" multiValued="false"/>
<field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
<!-- Copy the title field into titleString, and treat as a string
(rather than text type). This allows us to sort on the titleString -->
<field name="title_string" type="string" indexed="true" stored="false" />
<field name="data_dict" type="string" indexed="false" stored="true" />
<field name="validated_data_dict" type="string" indexed="false" stored="true" />
<field name="_version_" type="string" indexed="true" stored="true"/>
<dynamicField name="*_date" type="date" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="res_extras_*" type="text" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="vocab_*" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*" type="string" indexed="true" stored="false"/>
</fields>
<uniqueKey>index_id</uniqueKey>
<copyField source="url" dest="urls"/>
<copyField source="title" dest="title_ngram"/>
<copyField source="name" dest="name_ngram"/>
<copyField source="ckan_url" dest="urls"/>
<copyField source="download_url" dest="urls"/>
<copyField source="res_url" dest="urls"/>
<copyField source="extras_*" dest="text"/>
<copyField source="res_extras_*" dest="text"/>
<copyField source="vocab_*" dest="text"/>
<copyField source="urls" dest="text"/>
<copyField source="name" dest="text"/>
<copyField source="title" dest="text"/>
<copyField source="text" dest="text"/>
<copyField source="license" dest="text"/>
<copyField source="notes" dest="text"/>
<copyField source="tags" dest="text"/>
<copyField source="groups" dest="text"/>
<copyField source="organization" dest="text"/>
<copyField source="res_name" dest="text"/>
<copyField source="res_description" dest="text"/>
<copyField source="maintainer" dest="text"/>
<copyField source="author" dest="text"/>
</schema>

2
compose/2.9/.ckan-env
View File

@ -36,5 +36,5 @@ CKAN__DATAPUSHER__URL=http://datapusher:8000
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
# Solr configuration # Solr configuration
CKAN_VERSION=2.9.7 CKAN_VERSION=2.9.9
CKAN_CORE_NAME=ckan CKAN_CORE_NAME=ckan

2
compose/2.9/.env
View File

@ -12,7 +12,7 @@ POSTGRES_PORT=5432
DATASTORE_READONLY_PASSWORD=datastore DATASTORE_READONLY_PASSWORD=datastore
# CKAN # CKAN
CKAN_VERSION=2.9.7 CKAN_VERSION=2.9.9
CKAN_SITE_ID=default CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000 CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000 CKAN_PORT=5000

2
compose/2.9/solr8/ckan_init_solr.sh
View File

@ -4,7 +4,7 @@
# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION # Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION
# Example: # Example:
# CKAN_CORE_NAME=ckan # CKAN_CORE_NAME=ckan
# CKAN_VERSION=2.9.7 # CKAN_VERSION=2.9.9
set -e set -e

2
examples/harvest/.ckan-env
View File

@ -33,5 +33,5 @@ CKAN__HARVEST__MQ__TYPE=redis
CKAN__HARVEST__MQ__HOSTNAME=redis CKAN__HARVEST__MQ__HOSTNAME=redis
# Solr configuration # Solr configuration
CKAN_VERSION=2.9.7 CKAN_VERSION=2.9.9
CKAN_CORE_NAME=ckan CKAN_CORE_NAME=ckan

2
examples/harvest/.env
View File

@ -12,7 +12,7 @@ POSTGRES_PORT=5432
DATASTORE_READONLY_PASSWORD=datastore DATASTORE_READONLY_PASSWORD=datastore
# CKAN # CKAN
CKAN_VERSION=2.9.7 CKAN_VERSION=2.9.9
CKAN_SITE_ID=default CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000 CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000 CKAN_PORT=5000

18
examples/harvest/Dockerfile
View File

@ -1,7 +1,7 @@
################### ###################
### Extensions #### ### Extensions ####
################### ###################
FROM ghcr.io/keitaroinc/ckan:2.9.7 as extbuild FROM ghcr.io/keitaroinc/ckan:2.9.9 as extbuild
# Locations and tags, please use specific tags or revisions # Locations and tags, please use specific tags or revisions
ENV HARVEST_GIT_URL=https://github.com/ckan/ckanext-harvest ENV HARVEST_GIT_URL=https://github.com/ckan/ckanext-harvest
@ -12,13 +12,13 @@ USER root
# Install necessary packages to build extensions # Install necessary packages to build extensions
RUN apk add --no-cache \ RUN apk add --no-cache \
python3-dev \ python3-dev \
gcc \ gcc \
g++ \ g++ \
libffi-dev \ libffi-dev \
openssl-dev \ openssl-dev \
rust \ rust \
cargo cargo
# Fetch and build the custom CKAN extensions # Fetch and build the custom CKAN extensions
RUN pip wheel --wheel-dir=/wheels git+${HARVEST_GIT_URL}@${HARVEST_GIT_BRANCH}#egg=ckanext-harvest RUN pip wheel --wheel-dir=/wheels git+${HARVEST_GIT_URL}@${HARVEST_GIT_BRANCH}#egg=ckanext-harvest
@ -30,7 +30,7 @@ USER ckan
############ ############
### MAIN ### ### MAIN ###
############ ############
FROM ghcr.io/keitaroinc/ckan:2.9.7 FROM ghcr.io/keitaroinc/ckan:2.9.9
LABEL maintainer="Keitaro Inc <info@keitaro.com>" LABEL maintainer="Keitaro Inc <info@keitaro.com>"

2
examples/s3filestore/.ckan-env
View File

@ -38,5 +38,5 @@ CKANEXT__S3FILESTORE__REGION_NAME=us-east-1
CKANEXT__S3FILESTORE__SIGNATURE_VERSION=s3v4 CKANEXT__S3FILESTORE__SIGNATURE_VERSION=s3v4
# Solr configuration # Solr configuration
CKAN_VERSION=2.9.7 CKAN_VERSION=2.9.9
CKAN_CORE_NAME=ckan CKAN_CORE_NAME=ckan

2
examples/s3filestore/.env
View File

@ -12,7 +12,7 @@ POSTGRES_PORT=5432
DATASTORE_READONLY_PASSWORD=datastore DATASTORE_READONLY_PASSWORD=datastore
# CKAN # CKAN
CKAN_VERSION=2.9.7 CKAN_VERSION=2.9.9
CKAN_SITE_ID=default CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5000 CKAN_SITE_URL=http://localhost:5000
CKAN_PORT=5000 CKAN_PORT=5000

4
examples/s3filestore/Dockerfile
View File

@ -1,7 +1,7 @@
################### ###################
### Extensions #### ### Extensions ####
################### ###################
FROM ghcr.io/keitaroinc/ckan:2.9.7 as extbuild FROM ghcr.io/keitaroinc/ckan:2.9.9 as extbuild
# Locations and tags, please use specific tags or revisions # Locations and tags, please use specific tags or revisions
ENV S3FILESTORE_GIT_URL=https://github.com/keitaroinc/ckanext-s3filestore ENV S3FILESTORE_GIT_URL=https://github.com/keitaroinc/ckanext-s3filestore
@ -20,7 +20,7 @@ USER ckan
############ ############
### MAIN ### ### MAIN ###
############ ############
FROM ghcr.io/keitaroinc/ckan:2.9.7 FROM ghcr.io/keitaroinc/ckan:2.9.9
LABEL maintainer="Keitaro Inc <info@keitaro.com>" LABEL maintainer="Keitaro Inc <info@keitaro.com>"

3
images/ckan/2.7/Dockerfile
View File

@ -170,6 +170,9 @@ RUN pip install -e /srv/app/src/ckan && \
# Remove wheels # Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
# Install python2 secrets for generating sessions
RUN pip install python2-secrets
# Copy necessary scripts # Copy necessary scripts
COPY setup/app ${APP_DIR} COPY setup/app ${APP_DIR}

1
images/ckan/2.7/Dockerfile.focal
View File

@ -205,6 +205,7 @@ RUN pip install -e /srv/app/src/ckan && \
# Create and update CKAN config # Create and update CKAN config
# Generate CKAN config # Generate CKAN config
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \ paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Set the default level for extensions to INFO # Set the default level for extensions to INFO
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \

8
images/ckan/2.7/setup/app/start_ckan.sh
View File

@ -12,6 +12,14 @@ then
done done
fi fi
# Add session secret from chart
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET ]];then
echo "Not all environment variables are set. Generating sessions..."
else
echo "Setting session secrets from environment variables"
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
fi
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
then then
echo "Setting beaker.session.secret in ini file" echo "Setting beaker.session.secret in ini file"

3
images/ckan/2.8/Dockerfile
View File

@ -163,6 +163,9 @@ RUN pip install -e /srv/app/src/ckan && \
# Remove wheels # Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
# Install python2 secrets for generating sessions
RUN pip install python2-secrets
# Copy necessary scripts # Copy necessary scripts
COPY setup/app ${APP_DIR} COPY setup/app ${APP_DIR}

1
images/ckan/2.8/Dockerfile.focal
View File

@ -191,6 +191,7 @@ RUN pip install -e /srv/app/src/ckan && \
# Create and update CKAN config # Create and update CKAN config
# Generate CKAN config # Generate CKAN config
paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \ paster --plugin=ckan make-config ckan ${APP_DIR}/production.ini && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Set the default level for extensions to INFO # Set the default level for extensions to INFO
paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ paster --plugin=ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \

8
images/ckan/2.8/setup/app/start_ckan.sh
View File

@ -12,6 +12,14 @@ then
done done
fi fi
# Add session secret from chart
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET ]];then
echo "Not all environment variables are set. Generating sessions..."
else
echo "Setting session secrets from environment variables"
paster --plugin=ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
fi
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
then then
echo "Setting beaker.session.secret in ini file" echo "Setting beaker.session.secret in ini file"

84
images/ckan/2.9/Dockerfile
View File

@ -4,11 +4,11 @@
FROM alpine:3.13.7 as ckanbuild FROM alpine:3.13.7 as ckanbuild
# Used by Github Actions to tag the image with # Used by Github Actions to tag the image with
ENV IMAGE_TAG=2.9.7 ENV IMAGE_TAG=2.9.9
# Set CKAN version to build # Set CKAN version to build
ENV GIT_URL=https://github.com/ckan/ckan.git ENV GIT_URL=https://github.com/ckan/ckan.git
ENV GIT_BRANCH=ckan-2.9.7 ENV GIT_BRANCH=ckan-2.9.9
# Set src dirs # Set src dirs
ENV SRC_DIR=/srv/app/src ENV SRC_DIR=/srv/app/src
@ -18,25 +18,25 @@ WORKDIR ${SRC_DIR}
# Packages to build CKAN requirements and plugins # Packages to build CKAN requirements and plugins
RUN apk add --no-cache \ RUN apk add --no-cache \
python3 \ python3 \
python3-dev \ python3-dev \
git \ git \
curl \ curl \
postgresql-dev \ postgresql-dev \
linux-headers \ linux-headers \
gcc \ gcc \
make \ make \
g++ \ g++ \
autoconf \ autoconf \
automake \ automake \
libtool \ libtool \
patch \ patch \
musl-dev \ musl-dev \
pcre-dev \ pcre-dev \
pcre \ pcre \
libffi-dev \ libffi-dev \
libxml2-dev \ libxml2-dev \
libxslt-dev libxslt-dev
# Link python to python3 # Link python to python3
RUN ln -s /usr/bin/python3 /usr/bin/python RUN ln -s /usr/bin/python3 /usr/bin/python
@ -57,12 +57,17 @@ RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
COPY ./patches ${SRC_DIR}/patches COPY ./patches ${SRC_DIR}/patches
COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
# Apply patches # Apply patches
RUN ${SRC_DIR}/apply_ckan_patches.sh RUN cd ${SRC_DIR} && ls -lah ${SRC_DIR} && ash ${SRC_DIR}/apply_ckan_patches.sh
RUN rm -rf /srv/app/src/ckan/.git RUN rm -rf /srv/app/src/ckan/.git
# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736
RUN echo 'Cython < 3.0' > /tmp/constraint.txt
RUN PIP_CONSTRAINT=/tmp/constraint.txt pip wheel --wheel-dir=/wheels PyYAML==5.4.1
# RUN pip-compile ckan/requirements.in
RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3
########################### ###########################
### Default-Extensions #### ### Default-Extensions ####
########################### ###########################
@ -80,10 +85,10 @@ ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
ENV ENVVARS_GIT_BRANCH=0.0.1 ENV ENVVARS_GIT_BRANCH=0.0.1
RUN apk add --no-cache \ RUN apk add --no-cache \
python3 \ python3 \
python3-dev \ python3-dev \
git \ git \
curl curl
# Link python to python3 # Link python to python3
RUN ln -s /usr/bin/python3 /usr/bin/python RUN ln -s /usr/bin/python3 /usr/bin/python
@ -116,18 +121,18 @@ ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
# Install necessary packages to run CKAN # Install necessary packages to run CKAN
RUN apk add --no-cache \ RUN apk add --no-cache \
python3 \ python3 \
bash \ bash \
git \ git \
gettext \ gettext \
curl \ curl \
postgresql-client \ postgresql-client \
libmagic \ libmagic \
pcre \ pcre \
libxslt \ libxslt \
libxml2 \ libxml2 \
tzdata \ tzdata \
apache2-utils && \ apache2-utils && \
# Create SRC_DIR # Create SRC_DIR
mkdir -p ${SRC_DIR} && \ mkdir -p ${SRC_DIR} && \
# Link python to python3 # Link python to python3
@ -174,6 +179,7 @@ RUN pip install -e /srv/app/src/ckan && \
# Change ownership to app user # Change ownership to app user
chown -R ckan:ckan /srv/app chown -R ckan:ckan /srv/app
RUN pip install sqlalchemy==1.3.19
# Remove wheels # Remove wheels
RUN rm -rf /srv/app/wheels /srv/app/ext_wheels RUN rm -rf /srv/app/wheels /srv/app/ext_wheels

85
images/ckan/2.9/Dockerfile.focal
View File

@ -4,11 +4,11 @@
FROM ubuntu:focal-20210827 as ckanbuild FROM ubuntu:focal-20210827 as ckanbuild
# Used by Github Actions to tag the image with # Used by Github Actions to tag the image with
ENV IMAGE_TAG=2.9.7-focal ENV IMAGE_TAG=2.9.9-focal
# Set CKAN version to build # Set CKAN version to build
ENV GIT_URL=https://github.com/ckan/ckan.git ENV GIT_URL=https://github.com/ckan/ckan.git
ENV GIT_BRANCH=ckan-2.9.7 ENV GIT_BRANCH=ckan-2.9.9
# Set timezone # Set timezone
ENV TZ=UTC ENV TZ=UTC
@ -32,29 +32,29 @@ RUN update-locale LANG=${LC_ALL}
# Instal apt-utils # Instal apt-utils
RUN apt-get install --no-install-recommends -y \ RUN apt-get install --no-install-recommends -y \
apt-utils apt-utils
# Packages to build CKAN requirements and plugins # Packages to build CKAN requirements and plugins
RUN apt-get install --no-install-recommends -y \ RUN apt-get install --no-install-recommends -y \
git \ git \
curl \ curl \
ca-certificates \ ca-certificates \
python3 \ python3 \
libpq-dev \ libpq-dev \
linux-headers-generic \ linux-headers-generic \
gcc-10 \ gcc-10 \
make \ make \
g++-10 \ g++-10 \
autoconf \ autoconf \
automake \ automake \
libtool \ libtool \
patch \ patch \
libpcre3-dev \ libpcre3-dev \
libpcre3 \ libpcre3 \
python3-dev \ python3-dev \
libffi-dev \ libffi-dev \
libxml2-dev \ libxml2-dev \
libxslt-dev libxslt-dev
# Use gcc 10 # Use gcc 10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10
@ -102,11 +102,11 @@ ENV ENVVARS_GIT_BRANCH=0.0.1
RUN apt-get update && \ RUN apt-get update && \
apt-get install --no-install-recommends -y \ apt-get install --no-install-recommends -y \
git \ git \
curl \ curl \
ca-certificates \ ca-certificates \
python3 \ python3 \
python3-dev python3-dev
# Link python to python3 # Link python to python3
RUN ln -s /usr/bin/python3 /usr/bin/python RUN ln -s /usr/bin/python3 /usr/bin/python
@ -158,21 +158,21 @@ ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
# Install necessary packages to run CKAN # Install necessary packages to run CKAN
RUN apt-get update && \ RUN apt-get update && \
apt-get install --no-install-recommends -y \ apt-get install --no-install-recommends -y \
gettext \ gettext \
curl \ curl \
ca-certificates \ ca-certificates \
libpq5 \ libpq5 \
git \ git \
postgresql-client \ postgresql-client \
python3 \ python3 \
python3-distutils \ python3-distutils \
libpython3.8 \ libpython3.8 \
libmagic1 \ libmagic1 \
libpcre3 \ libpcre3 \
libxslt1.1 \ libxslt1.1 \
libxml2 \ libxml2 \
tzdata \ tzdata \
apache2-utils && \ apache2-utils && \
rm -rf /var/lib/apt/lists/* && \ rm -rf /var/lib/apt/lists/* && \
# Create SRC_DIR # Create SRC_DIR
mkdir -p ${SRC_DIR} && \ mkdir -p ${SRC_DIR} && \
@ -210,6 +210,7 @@ RUN pip install -e /srv/app/src/ckan && \
# Generate CKAN config # Generate CKAN config
ckan generate config ${APP_DIR}/production.ini && \ ckan generate config ${APP_DIR}/production.ini && \
# Configure plugins # Configure plugins
ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
# Create the data directory # Create the data directory
mkdir ${DATA_DIR} && \ mkdir ${DATA_DIR} && \

10
images/ckan/2.9/patches/00_patch_sql_url.patch
View File

@ -1,11 +1,11 @@
--- ckan/ckan/model/__init__.py 2021-02-16 14:47:06.168327441 +0100 --- ckan/ckan/model/__init__.py
+++ ckan/ckan/model/__init__.py 2021-02-16 14:48:00.740780218 +0100 +++ ckan/ckan/model/__init__.py
@@ -266,7 +266,7 @@ @@ -276,7 +276,7 @@ class Repository():
self.reset_alembic_output() self.reset_alembic_output()
alembic_config = AlembicConfig(self._alembic_ini) alembic_config = AlembicConfig(self._alembic_ini)
alembic_config.set_main_option( alembic_config.set_main_option(
- "sqlalchemy.url", str(self.metadata.bind.url) - "sqlalchemy.url", config.get("sqlalchemy.url")
+ "sqlalchemy.url", str(self.metadata.bind.url).replace('%', '%%') + "sqlalchemy.url", config.get("sqlalchemy.url").replace('%', '%%')
) )
try: try:
sqlalchemy_migrate_version = self.metadata.bind.execute( sqlalchemy_migrate_version = self.metadata.bind.execute(

6
images/ckan/2.9/patches/02_patch_postgres_username_split.patch → images/ckan/2.9/patches/01_patch_postgres_username_split.patch
View File

@ -1,6 +1,6 @@
--- ckan/ckanext/datastore/backend/postgres.py 2021-02-18 11:01:56.692267462 +0100 --- ckan/ckanext/datastore/backend/postgres.py
+++ ckan/ckanext/datastore/backend/postgres-patch.py 2021-02-18 13:45:16.033193435 +0100 +++ ckan/ckanext/datastore/backend/postgres.py
@@ -1690,7 +1690,7 @@ @@ -1809,7 +1809,7 @@ class DatastorePostgresqlBackend(DatastoreBackend):
read only user. read only user.
''' '''
write_connection = self._get_write_engine().connect() write_connection = self._get_write_engine().connect()

11
images/ckan/2.9/patches/01_patch_resource_replace_upload.patch
View File

@ -1,11 +0,0 @@
--- ckan/ckan/logic/action/update.py 2021-02-17 16:46:55.673578728 +0100
+++ ckan/ckan/logic/action/update-edit.py 2021-02-17 16:47:28.905879170 +0100
@@ -929,7 +929,7 @@
'''
model = context['model']
- session = model.Session
+ session = model.meta.create_local_session()
context['session'] = session
user = context['user']

39
images/ckan/2.9/setup/app/basic-auth-uwsgi.conf Normal file
View File

@ -0,0 +1,39 @@
[uwsgi]
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
socket = /tmp/uwsgi.sock
uid = ckan
gid = ckan
http = :5000
master = true
enable-threads = true
lazy-apps = true
gevent-early-monkey-patch = true
vacuum = true
single-interpreter= true
die-on-term = true
need-app = true
auto-procname = true
wsgi-file = /srv/app/wsgi.py
module = wsgi:application
gevent = 2000
callable = application
paste = config:/srv/app/production.ini
paste-logger = /srv/app/production.ini
post-buffering = 1
buffer-size= 12288
max-requests = 3000
max-worker-lifetime = 3600
reload-on-rss = 4096
worker-reload-mercy = 60
socket-timeout = 300
queue = 1000
queue-blocksize = 204800
static-gzip-all = true
listen = 1000
http-timeout = 1000
http-headers-timeout = 1000
http-connect-timeout = 1000

20
images/ckan/2.9/setup/app/prerun.py
View File

@ -1,12 +1,9 @@
""" """
Copyright (c) 2016 Keitaro AB Copyright (c) 2016 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
You may obtain a copy of the License at You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0 https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -14,12 +11,13 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
""" """
from multiprocessing import connection
import os import os
import sys import sys
import subprocess import subprocess
import psycopg2 import psycopg2
from sqlalchemy.engine.url import make_url from sqlalchemy.engine.url import make_url
import urllib.request, urllib.error, urllib.parse import urllib.request, urllib.error, urllib.parse, base64
import re import re
import json import json
@ -70,10 +68,20 @@ def check_solr_connection(retry=None):
sys.exit(1) sys.exit(1)
url = os.environ.get('CKAN_SOLR_URL', '') url = os.environ.get('CKAN_SOLR_URL', '')
username = os.environ.get('CKAN_SOLR_USER', '')
password = os.environ.get('CKAN_SOLR_PASSWORD', '')
search_url = '{url}/schema/name?wt=json'.format(url=url) search_url = '{url}/schema/name?wt=json'.format(url=url)
try: try:
connection = urllib.request.urlopen(search_url) if not username:
connection = urllib.request.urlopen(search_url)
else:
request = urllib.request.Request(search_url)
base64string = base64.b64encode(bytes('%s:%s' % (username, password),'ascii'))
request.add_header("Authorization", "Basic %s" % base64string.decode('utf-8'))
connection = urllib.request.urlopen(request)
except urllib.error.URLError as e: except urllib.error.URLError as e:
print('[prerun] Unable to connect to solr...try again in a while.') print('[prerun] Unable to connect to solr...try again in a while.')
import time import time
@ -217,4 +225,4 @@ if __name__ == '__main__':
init_db() init_db()
if os.environ.get('CKAN_DATASTORE_WRITE_URL'): if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
init_datastore() init_datastore()
create_sysadmin() create_sysadmin()

24
images/ckan/2.9/setup/app/start_ckan.sh
View File

@ -12,17 +12,26 @@ then
done done
fi fi
# Add session secret from chart
if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET || -z $JWT_ENCODE_SECRET || -v $JWT_ENCODE_SECRET || -z $JWT_DECODE_SECRET || -v $JWT_DECODE_SECRET ]];then
echo "Not all environment variables are set. Generating sessions..."
else
echo "Setting session secrets from environment variables"
ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET"
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_ENCODE_SECRET"
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_DECODE_SECRET"
fi
if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini
then then
echo "Setting secrets in ini file" echo "Setting secrets in ini file"
ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')" ckan config-tool $APP_DIR/production.ini "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')" JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_SECRET"
ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_SECRET"
fi fi
echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid ckan --gid ckan --http :5000 --master --enable-threads --wsgi-file /srv/app/wsgi.py --module wsgi:application --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch --vacuum --harakiri 50 --callable application"
# Run the prerun script to init CKAN and create the default admin user # Run the prerun script to init CKAN and create the default admin user
python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; } python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; }
@ -52,14 +61,15 @@ then
# Generate htpasswd file for basicauth # Generate htpasswd file for basicauth
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
# Start uwsgi with basicauth # Start uwsgi with basicauth
uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS uwsgi --ini /srv/app/basic-auth-uwsgi.conf -p ${UWSGI_PROC_NO:-2} --pcre-jit
else else
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
exit 1 exit 1
fi fi
else else
# Start uwsgi # Start uwsgi
uwsgi $UWSGI_OPTS echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"
uwsgi --ini /srv/app/uwsgi.conf -p ${UWSGI_PROC_NO:-2}
fi fi
else else
echo "[prerun] failed...not starting CKAN." echo "[prerun] failed...not starting CKAN."

37
images/ckan/2.9/setup/app/uwsgi.conf
View File

@ -1,2 +1,37 @@
[uwsgi] [uwsgi]
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd socket = /tmp/uwsgi.sock
uid = ckan
gid = ckan
http = :5000
master = true
enable-threads = true
lazy-apps = true
gevent-early-monkey-patch = true
vacuum = true
single-interpreter= true
die-on-term = true
need-app = true
auto-procname = true
wsgi-file = /srv/app/wsgi.py
module = wsgi:application
gevent = 2000
callable = application
paste = config:/srv/app/production.ini
paste-logger = /srv/app/production.ini
post-buffering = 1
buffer-size= 12288
max-requests = 3000
max-worker-lifetime = 3600
reload-on-rss = 4096
worker-reload-mercy = 60
socket-timeout = 300
queue = 1000
queue-blocksize = 204800
static-gzip-all = true
listen = 1000
http-timeout = 1000
http-headers-timeout = 1000
http-connect-timeout = 1000

21
images/psql-init/Dockerfile Normal file
View File

@ -0,0 +1,21 @@
# Start with a lightweight base image
FROM python:3.9-alpine
# Used by Github Actions to tag the image with
ENV IMAGE_TAG=0.0.1
# Set the working directory in the container
WORKDIR /srv
# Copy the requirements file to the container
COPY requirements.txt .
# Install the Python dependencies
RUN pip install --no-cache-dir -r requirements.txt
# Copy the rest of the application code to the container
COPY psql-init/ .
CMD ["python", "/srv/psql-init.py"]

277
images/psql-init/psql-init/psql-init.py Normal file
View File

@ -0,0 +1,277 @@
"""
Copyright (c) 2020 Keitaro AB
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import sys
import subprocess
import re
import psycopg2
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
from psycopg2.extensions import AsIs
from sqlalchemy.engine.url import make_url
ckan_conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '')
datastorerw_conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL', '')
datastorero_conn_str = os.environ.get('CKAN_DATASTORE_READ_URL', '')
master_user = os.environ.get('PSQL_MASTER', '')
master_passwd = os.environ.get('PSQL_PASSWD', '')
master_database = os.environ.get('PSQL_DB', '')
class DB_Params:
def __init__(self, conn_str):
self.db_user = make_url(conn_str).username
self.db_passwd = make_url(conn_str).password
self.db_host = make_url(conn_str).host
self.db_name = make_url(conn_str).database
def check_db_connection(db_params, retry=None):
print('Checking whether database is up...')
if retry is None:
retry = 20
elif retry == 0:
print('Giving up...')
sys.exit(1)
try:
con = psycopg2.connect(user=master_user,
host=db_params.db_host,
password=master_passwd,
database=master_database)
except psycopg2.Error as e:
print((str(e)))
print('Unable to connect to the database...try again in a while.')
import time
time.sleep(30)
check_db_connection(db_params, retry=retry - 1)
else:
con.close()
def create_user(db_params):
con = None
try:
con = psycopg2.connect(user=master_user,
host=db_params.db_host,
password=master_passwd,
database=master_database)
con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = con.cursor()
print("Creating user " + db_params.db_user.split("@")[0])
cur.execute('CREATE ROLE "%s" ' +
'WITH ' +
'LOGIN NOSUPERUSER INHERIT ' +
'CREATEDB NOCREATEROLE NOREPLICATION ' +
'PASSWORD %s',
(AsIs(db_params.db_user.split("@")[0]),
db_params.db_passwd,))
except(Exception, psycopg2.DatabaseError) as error:
print("ERROR DB: ", error)
finally:
cur.close()
con.close()
def create_db(db_params):
con = None
try:
con = psycopg2.connect(user=master_user,
host=db_params.db_host,
password=master_passwd,
database=master_database)
con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = con.cursor()
cur.execute('GRANT "' + db_params.db_user.split("@")
[0] + '" TO "' + master_user.split("@")[0] + '"')
print("Creating database " + db_params.db_name + " with owner " +
db_params.db_user.split("@")[0])
cur.execute('CREATE DATABASE ' + db_params.db_name + ' OWNER "' +
db_params.db_user.split("@")[0] + '"')
cur.execute('GRANT ALL PRIVILEGES ON DATABASE ' +
db_params.db_name + ' TO "' +
db_params.db_user.split("@")[0] + '"')
if is_pg_buffercache_enabled(db_params) >= 1:
# FIXME: This is a known issue with pg_buffercache access
# For more info check this thread:
# https://www.postgresql.org/message-id/21009351582737086%40iva6-22e79380f52c.qloud-c.yandex.net
print("Granting privileges on pg_monitor to " +
db_params.db_user.split("@")[0])
cur.execute('GRANT "pg_monitor" TO "' + db_params.db_user.split("@")[0] + '"')
except(Exception, psycopg2.DatabaseError) as error:
print("ERROR DB: ", error)
finally:
cur.close()
con.close()
def is_pg_buffercache_enabled(db_params):
con = None
result = None
try:
con = psycopg2.connect(user=master_user,
host=db_params.db_host,
password=master_passwd,
database=db_params.db_name)
con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = con.cursor()
cur.execute("SELECT count(*) FROM pg_extension " +
"WHERE extname = 'pg_buffercache'")
result = cur.fetchone()
except(Exception, psycopg2.DatabaseError) as error:
print("ERROR DB: ", error)
finally:
cur.close()
con.close()
return result[0]
def set_datastore_permissions(datastore_rw_params, datastore_ro_params, sql):
con = None
try:
con = psycopg2.connect(user=master_user,
host=datastore_rw_params.db_host,
password=master_passwd,
database=datastore_rw_params.db_name)
con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = con.cursor()
cur.execute('GRANT CONNECT ON DATABASE ' +
datastore_rw_params.db_name +
' TO ' + datastore_ro_params.db_user.split("@")[0])
if is_pg_buffercache_enabled(datastore_rw_params) >= 1:
print("Granting privileges on pg_monitor to " +
datastore_ro_params.db_user.split("@")[0])
cur.execute('GRANT ALL PRIVILEGES ON TABLE pg_monitor TO ' +
datastore_ro_params.db_user.split("@")[0])
print("Setting datastore permissions\n")
print(sql)
cur.execute(sql)
print("Datastore permissions applied.")
except Exception as error:
print("ERROR DB: ", error)
finally:
cur.close()
con.close()
if master_user == '' or master_passwd == '' or master_database == '':
print("No master postgresql user provided.")
print("Cannot initialize default CKAN db resources. Exiting!")
sys.exit(1)
print("Master DB: " + master_database + " Master User: " + master_user)
ckan_db = DB_Params(ckan_conn_str)
datastorerw_db = DB_Params(datastorerw_conn_str)
datastorero_db = DB_Params(datastorero_conn_str)
# Check to see whether we can connect to the database, exit after 10 mins
check_db_connection(ckan_db)
try:
create_user(ckan_db)
except(Exception, psycopg2.DatabaseError) as error:
print("ERROR DB: ", error)
try:
create_user(datastorerw_db)
except(Exception, psycopg2.DatabaseError) as error:
print("ERROR DB: ", error)
try:
create_user(datastorero_db)
except(Exception, psycopg2.DatabaseError) as error:
print("ERROR DB: ", error)
try:
create_db(ckan_db)
except(Exception, psycopg2.DatabaseError) as error:
print("ERROR DB: ", error)
try:
create_db(datastorerw_db)
except(Exception, psycopg2.DatabaseError) as error:
print("ERROR DB: ", error)
def execute_sql_script(ckan_dbp, datastorero_dbp, datastorerw_dbp, script_path):
# Connect to the database
conn = psycopg2.connect(
user=master_user,
host=datastorerw_dbp.db_host,
password=master_passwd,
database=datastorerw_dbp.db_name
)
try:
# Create a cursor
cur = conn.cursor()
# Execute the SQL script
with open(script_path, 'r') as f:
sql_script = f.read()
# Replace placeholders with actual values
sql_script = sql_script.replace('{datastoredb}', datastorerw_dbp.db_name)
sql_script = sql_script.replace('{readuser}', datastorero_dbp.db_user)
sql_script = sql_script.replace('{writeuser}', datastorerw_dbp.db_user)
sql_script = sql_script.replace('{mainuser}', ckan_dbp.db_user)
sql_script = sql_script.replace('{maindb}', ckan_dbp.db_name)
print("CKAN DB User:", ckan_dbp.db_user)
# Execute the SQL script
cur.execute(sql_script)
# Commit the changes
conn.commit()
print("SQL script executed successfully.")
print("CKAN DB User:", ckan_dbp.db_user)
print("read/write DB User:", datastorerw_dbp.db_user)
print("read/write DB name:", datastorerw_dbp.db_name)
print("read/write host:", datastorerw_dbp.db_host)
print("read DB user:", datastorero_dbp.db_user)
print("read DB name:", datastorero_dbp.db_name)
except psycopg2.Error as e:
print(f"Error executing SQL script: {str(e)}")
finally:
# Close the cursor and the connection
cur.close()
conn.close()
set_permissions = './set_permissions.sql'
# Print the current working directory
print("Current working directory:", os.getcwd())
# Check if the file exists
if os.path.isfile(set_permissions):
print("File exists.")
# Call the execute_sql_script function with the appropriate arguments
execute_sql_script(ckan_db, datastorero_db, datastorerw_db, set_permissions)
else:
print("File not found.")

108
images/psql-init/psql-init/set_permissions.sql Normal file
View File

@ -0,0 +1,108 @@
/*
This script configures the permissions for the datastore.
It ensures that the datastore read-only user will only be able to select from
the datastore database but has no create/write/edit permission or any
permissions on other databases. You must execute this script as a database
superuser on the PostgreSQL server that hosts your datastore database.
For example, if PostgreSQL is running locally and the "postgres" user has the
appropriate permissions (as in the default Ubuntu PostgreSQL install), you can
run:
ckan -c /etc/ckan/default/ckan.ini datastore set-permissions | sudo -u postgres psql
Or, if your PostgreSQL server is remote, you can pipe the permissions script
over SSH:
ckan -c /etc/ckan/default/ckan.ini datastore set-permissions | ssh dbserver sudo -u postgres psql
*/
-- Most of the following commands apply to an explicit database or to the whole
-- 'public' schema, and could be executed anywhere. But ALTER DEFAULT
-- PERMISSIONS applies to the current database, and so we must be connected to
-- the datastore DB:
--\connect {datastoredb}
-- revoke permissions for the read-only user
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
REVOKE USAGE ON SCHEMA public FROM PUBLIC;
GRANT CREATE ON SCHEMA public TO {mainuser};
GRANT USAGE ON SCHEMA public TO {mainuser};
GRANT CREATE ON SCHEMA public TO {writeuser};
GRANT USAGE ON SCHEMA public TO {writeuser};
-- take connect permissions from main db
REVOKE CONNECT ON DATABASE {maindb} FROM {readuser};
-- grant select permissions for read-only user
GRANT CONNECT ON DATABASE {datastoredb} TO {readuser};
GRANT USAGE ON SCHEMA public TO {readuser};
-- grant access to current tables and views to read-only user
GRANT SELECT ON ALL TABLES IN SCHEMA public TO {readuser};
-- grant access to new tables and views by default
ALTER DEFAULT PRIVILEGES FOR USER {writeuser} IN SCHEMA public
GRANT SELECT ON TABLES TO {readuser};
-- a view for listing valid table (resource id) and view names
CREATE OR REPLACE VIEW "_table_metadata" AS
SELECT DISTINCT
substr(md5(dependee.relname || COALESCE(dependent.relname, '')), 0, 17) AS "_id",
dependee.relname AS name,
dependee.oid AS oid,
dependent.relname AS alias_of
FROM
pg_class AS dependee
LEFT OUTER JOIN pg_rewrite AS r ON r.ev_class = dependee.oid
LEFT OUTER JOIN pg_depend AS d ON d.objid = r.oid
LEFT OUTER JOIN pg_class AS dependent ON d.refobjid = dependent.oid
WHERE
(dependee.oid != dependent.oid OR dependent.oid IS NULL) AND
-- is a table (from pg_tables view definition)
-- or is a view (from pg_views view definition)
(dependee.relkind = 'r'::"char" OR dependee.relkind = 'v'::"char")
AND dependee.relnamespace = (
SELECT oid FROM pg_namespace WHERE nspname='public')
ORDER BY dependee.oid DESC;
ALTER VIEW "_table_metadata" OWNER TO {writeuser};
GRANT SELECT ON "_table_metadata" TO {readuser};
-- _full_text fields are now updated by a trigger when set to NULL
CREATE OR REPLACE FUNCTION populate_full_text_trigger() RETURNS trigger
AS $body$
BEGIN
IF NEW._full_text IS NOT NULL THEN
RETURN NEW;
END IF;
NEW._full_text := (
SELECT to_tsvector(string_agg(value, ' '))
FROM json_each_text(row_to_json(NEW.*))
WHERE key NOT LIKE '\_%');
RETURN NEW;
END;
$body$ LANGUAGE plpgsql;
ALTER FUNCTION populate_full_text_trigger() OWNER TO {writeuser};
-- migrate existing tables that don't have full text trigger applied
DO $body$
BEGIN
EXECUTE coalesce(
(SELECT string_agg(
'CREATE TRIGGER zfulltext BEFORE INSERT OR UPDATE ON ' ||
quote_ident(relname) || ' FOR EACH ROW EXECUTE PROCEDURE ' ||
'populate_full_text_trigger();', ' ')
FROM pg_class
LEFT OUTER JOIN pg_trigger AS t
ON t.tgrelid = relname::regclass AND t.tgname = 'zfulltext'
WHERE relkind = 'r'::"char" AND t.tgname IS NULL
AND relnamespace = (
SELECT oid FROM pg_namespace WHERE nspname='public')),
'SELECT 1;');
END;
$body$;

2
images/psql-init/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
psycopg2-binary==2.9.3
sqlalchemy==1.3.5