Add s3filestore example

2020-09-11 12:14:39 +02:00 · 2020-09-11 12:14:39 +02:00 · 3f0ebece31
parent 348dfb841e
commit 3f0ebece31
6 changed files with 131 additions and 70 deletions
--- a/compose/.ckan-env
+++ b/compose/.ckan-env
@ -23,11 +23,3 @@ CKAN_SMTP_STARTTLS=True
 CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
-
-# S3/MINIO settings
-#CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID = MINIOACCESSKEY
-#CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY = MINIOSECRETKEY
-#CKANEXT__S3FILESTORE__AWS_BUCKET_NAME = ckan
-#CKANEXT__S3FILESTORE__HOST_NAME = http://minio:9000
-#CKANEXT__S3FILESTORE__REGION_NAME = us-east-1
-#CKANEXT__S3FILESTORE__SIGNATURE_VERSION = s3v4
--- a/examples/Dockerfile
+++ b/examples/Dockerfile
@ -1,52 +0,0 @@
-###################
-### Extensions ####
-###################
-FROM keitaro/ckan:2.9.0 as extbuild
-
-MAINTAINER Keitaro Inc <info@keitaro.com>
-
-# Locations and tags, please use specific tags or revisions
-ENV HARVEST_GIT_URL=https://github.com/ckan/ckanext-harvest
-ENV HARVEST_GIT_BRANCH=v1.3.1
-
-# Switch to the root user
-USER root
-
-# Install necessary packages to build extensions
-RUN apk add --no-cache \
-        gcc \
-        g++ \
-        libffi-dev \
-        openssl-dev \
-        python3-dev 
-
-# Fetch and build the custom CKAN extensions
-RUN pip wheel --wheel-dir=/wheels git+${HARVEST_GIT_URL}@${HARVEST_GIT_BRANCH}#egg=ckanext-harvest
-RUN pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${HARVEST_GIT_BRANCH}/pip-requirements.txt
-RUN curl -o /wheels/harvest.txt https://raw.githubusercontent.com/ckan/ckanext-harvest/${HARVEST_GIT_BRANCH}/pip-requirements.txt
-
-############
-### MAIN ###
-############
-FROM keitaro/ckan:2.9.0
-
-ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher harvest ckan_harvester
-
-# Switch to the root user
-USER root
-
-COPY --from=extbuild /wheels /srv/app/ext_wheels
-
-# Install and enable the custom extensions
-RUN pip install --no-index --find-links=/srv/app/ext_wheels ckanext-harvest && \
-    pip install --no-index --find-links=/srv/app/ext_wheels -r /srv/app/ext_wheels/harvest.txt && \
-    # Not working atm since ckan config tool tries to load config before executing config-tool, workaround
-    # ckan -c ${APP_DIR}/production.ini config-tool "ckan.plugins = ${CKAN__PLUGINS}" && \
-    sed -i "/ckan.plugins = envvars/c ckan.plugins = ${CKAN__PLUGINS}" ${APP_DIR}/production.ini && \
-    chown -R ckan:ckan /srv/app
-
-# Remove wheels
-RUN rm -rf /srv/app/ext_wheels
-
-# Switch to the ckan user
-USER ckan
--- a/examples/s3filestore/.ckan-env
+++ b/examples/s3filestore/.ckan-env
@ -0,0 +1,33 @@
+# Runtime configuration of CKAN enabled through ckanext-envvars
+# Information about how it works: https://github.com/okfn/ckanext-envvars
+# Note that variables here take presedence over build/up time variables in .env
+
+# General Settings
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5000
+CKAN_MAX_UPLOAD_SIZE_MB=10
+# CKAN Plugins
+CKAN__PLUGINS=envvars s3filestore image_view text_view recline_view datastore datapusher
+# CKAN requires storage path to be set in order for filestore to be enabled
+CKAN__STORAGE_PATH=/srv/app/data
+CKAN__WEBASSETS__PATH=/srv/app/data/webassets
+# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
+CKAN_SYSADMIN_NAME=sysadmin
+CKAN_SYSADMIN_PASSWORD=password
+CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
+
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+# S3/MINIO settings
+CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID = MINIOACCESSKEY
+CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY = MINIOSECRETKEY
+CKANEXT__S3FILESTORE__AWS_BUCKET_NAME = ckan
+CKANEXT__S3FILESTORE__HOST_NAME = http://minio:9000
+CKANEXT__S3FILESTORE__REGION_NAME = us-east-1
+CKANEXT__S3FILESTORE__SIGNATURE_VERSION = s3v4
--- a/examples/s3filestore/.env
+++ b/examples/s3filestore/.env
@ -0,0 +1,29 @@
+# Variables in this file will be used as build arguments when running
+# docker-compose build and docker-compose up
+# Verify correct substitution with "docker-compose config"
+# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
+# docker-compose down -v
+# docker-compose build
+# docker-compose up -d
+
+# Database
+POSTGRES_PASSWORD=ckan
+POSTGRES_PORT=5432
+DATASTORE_READONLY_PASSWORD=datastore
+
+# CKAN
+CKAN_VERSION=2.8.5
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5000
+CKAN_MAX_UPLOAD_SIZE_MB=10
+
+# Datapusher
+DATAPUSHER_VERSION=0.0.17
+DATAPUSHER_MAX_CONTENT_LENGTH=10485760
+DATAPUSHER_CHUNK_SIZE=16384
+DATAPUSHER_CHUNK_INSERT_ROWS=250
+DATAPUSHER_DOWNLOAD_TIMEOUT=30
+
+# Redis
+REDIS_VERSION=6.0.7
--- a/examples/s3filestore/Dockerfile
+++ b/examples/s3filestore/Dockerfile
@ -0,0 +1,43 @@
+###################
+### Extensions ####
+###################
+FROM keitaro/ckan:2.8.5 as extbuild
+
+MAINTAINER Keitaro Inc <info@keitaro.com>
+
+# Locations and tags, please use specific tags or revisions
+ENV S3FILESTORE_GIT_URL=https://github.com/keitaroinc/ckanext-s3filestore
+ENV S3FILESTORE_GIT_BRANCH=master
+
+# Switch to the root user
+USER root
+
+# Fetch and build the custom CKAN extensions
+RUN pip wheel --wheel-dir=/wheels git+${S3FILESTORE_GIT_URL}@${S3FILESTORE_GIT_BRANCH}#egg=ckanext-s3filestore
+RUN pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/keitaroinc/ckanext-s3filestore/${S3FILESTORE_GIT_BRANCH}/requirements.txt
+RUN curl -o /wheels/s3filestore.txt https://raw.githubusercontent.com/keitaroinc/ckanext-s3filestore/${S3FILESTORE_GIT_BRANCH}/requirements.txt
+
+############
+### MAIN ###
+############
+FROM keitaro/ckan:2.8.5
+
+MAINTAINER Keitaro Inc <info@keitaro.com>
+
+ENV CKAN__PLUGINS envvars s3filestore image_view text_view recline_view datastore datapusher
+
+# Switch to the root user
+USER root
+
+COPY --from=extbuild /wheels /srv/app/ext_wheels
+
+# Install and enable the custom extensions
+RUN pip install --no-index --find-links=/srv/app/ext_wheels ckanext-s3filestore && \
+    pip install --no-index --find-links=/srv/app/ext_wheels -r /srv/app/ext_wheels/s3filestore.txt && \
+    paster --plugin=ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
+    chown -R ckan:ckan /srv/app
+
+# Remove wheels
+RUN rm -rf /srv/app/ext_wheels
+
+USER ckan
--- a/examples/s3filestore/docker-compose.yml
+++ b/examples/s3filestore/docker-compose.yml
@ -9,12 +9,11 @@ volumes:
 services:
  ckan:
    container_name: ckan
-    image: keitaro/ckan:${CKAN_VERSION}
-    links:
-      - db
-      - solr
-      - redis
-      - minio
+    build:
+      context: .
+    networks:
+      - frontend
+      - backend
    depends_on:
      - db
    ports:
@ -36,8 +35,9 @@ services:
  datapusher:
    container_name: datapusher
    image: keitaro/ckan-datapusher:${DATAPUSHER_VERSION}
-    links:
-      - ckan
+    networks:
+      - frontend
+      - backend
    ports:
      - "8000:8000"
    environment:
@ -49,11 +49,13 @@ services:
  db:
    container_name: db
    build:
-      context: .
+      context: ../../compose
      dockerfile: postgresql/Dockerfile
      args:
        - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    networks:
+      - backend
    environment:
      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -65,20 +67,27 @@ services:
  solr:
    container_name: solr
    build:
-      context: .
+      context: ../../compose
      dockerfile: solr/Dockerfile
      args:
        - CKAN_VERSION=${CKAN_VERSION}
+    networks:
+      - backend
    volumes:
      - solr_data:/opt/solr/server/solr/ckan/data

  redis:
    container_name: redis
    image: redis:${REDIS_VERSION}
+    networks:
+      - backend

  minio:
    container_name: minio
    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
+    networks:
+      - backend
+      - frontend
    ports:
      - "0.0.0.0:9000:9000"
    environment:
@ -91,6 +100,8 @@ services:
  mc:
    container_name: mc
    image: minio/mc:RELEASE.2020-08-08T02-33-58Z
+    networks:
+      - backend
    depends_on:
      - minio
    entrypoint: >
@ -98,4 +109,9 @@ services:
      /usr/bin/mc config host rm local;
      /usr/bin/mc config host add --quiet --api s3v4 local http://minio:9000 minio ckan123;
      /usr/bin/mc mb --quiet local/ckan/;
+      /usr/bin/mc policy set public local/ckan;
      "
+
+networks:
+  frontend:
+  backend: