diff --git a/README.md b/README.md index 437e166..3e0cad0 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,9 @@ docker compose --env-file .docker.env up Check if CKAN was succesfuly started on http://localhost:5000. ### Configuration -In order to configure CKAN within docker-compose we use both build/up time variables loaded via the [.env](./compose/2.9/.env) file, and runtime variables loaded via the [.ckan-env](./compose/2.9/.ckan-env) file. +In order to configure CKAN within docker-compose we use both build/up time variables loaded via the [.env](./compose/2.10/.env) file, and runtime variables loaded via the [.ckan-env](./compose/2.10/.ckan-env) file. -Variables in the [.env](./compose/2.9/.env) file are loaded when running `docker-compose build` and `docker-compose up`, while variables in [.ckan-env](./compose/2.9/.ckan-env) file are used withing the CKAN container at runtime to configure CKAN and CKAN extensions using [ckanext-envvars](https://github.com/okfn/ckanext-envvars). +Variables in the [.env](./compose/2.10/.env) file are loaded when running `docker-compose build` and `docker-compose up`, while variables in [.ckan-env](./compose/2.10/.ckan-env) file are used withing the CKAN container at runtime to configure CKAN and CKAN extensions using [ckanext-envvars](https://github.com/okfn/ckanext-envvars). ## Extending CKAN docker images Check some examples of extending CKAN docker images in the [examples](./examples) directory. @@ -48,7 +48,7 @@ We recommend to use a multi-stage approach to extend the docker images that we p ################### ### Extensions #### ################### -FROM ghcr.io/keitaroinc/ckan:2.9.9 as extbuild +FROM ghcr.io/keitaroinc/ckan:2.10.4 as extbuild # Switch to the root user USER root @@ -64,7 +64,7 @@ RUN pip wheel --wheel-dir=/wheels git+https://github.com/acmecorp/ckanext-acme@0 ############ ### MAIN ### ############ -FROM ghcr.io/keitaroinc/ckan:2.9.9 +FROM ghcr.io/keitaroinc/ckan:2.10.4 # Add the custom extensions to the plugins list ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher acme diff --git a/compose/2.10/Dockerfile b/compose/2.10/Dockerfile index 9746d48..f67382e 100644 --- a/compose/2.10/Dockerfile +++ b/compose/2.10/Dockerfile @@ -23,6 +23,9 @@ USER root #COPY --from=extbuild /wheels /srv/app/ext_wheels +# ENV PROJ_DIR=/usr/local +# ENV PROJ_INCDIR=/usr/local/include + # Add requirements.txt to the image COPY requirements.txt /srv/app/requirements.txt diff --git a/compose/2.10/docker-compose.yml b/compose/2.10/docker-compose.yml index b8c889e..b554fa0 100644 --- a/compose/2.10/docker-compose.yml +++ b/compose/2.10/docker-compose.yml @@ -95,7 +95,7 @@ services: redis: platform: linux/amd64 container_name: redis - image: redis:${REDIS_VERSION} + image: arm64v8/redis:${REDIS_VERSION} networks: - backend diff --git a/compose/2.10/requirements.txt b/compose/2.10/requirements.txt index 9db4c49..12a3e5c 100644 --- a/compose/2.10/requirements.txt +++ b/compose/2.10/requirements.txt @@ -1,3 +1,17 @@ # Spatial --e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial --r https://raw.githubusercontent.com/ckan/ckanext-spatial/v2.1.1/requirements.txt +# -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial +# -r https://raw.githubusercontent.com/ckan/ckanext-spatial/v2.1.1/requirements.txt + +ckantoolkit +lxml>=2.3 +argparse +pyparsing>=2.1.10 +requests>=1.1.0 +# cython==0.29.36; python_version < '3.9' +# pyproj==2.6.1; python_version < '3.9' +pyproj==3.6.1 + +Shapely==2.0.1 +OWSLib==0.28.1 +geojson==3.0.1 + diff --git a/compose/2.10/solr8/ckan_init_solr.sh b/compose/2.10/solr8/ckan_init_solr.sh index 091450f..35f3c17 100755 --- a/compose/2.10/solr8/ckan_init_solr.sh +++ b/compose/2.10/solr8/ckan_init_solr.sh @@ -4,7 +4,7 @@ # Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION # Example: # CKAN_CORE_NAME=ckan -# CKAN_VERSION=2.9.7 +# CKAN_VERSION=2.10.4 set -e diff --git a/compose/2.10/src/ckanext-d4science/README.md b/compose/2.10/src/ckanext-d4science/README.md index 17f8649..d2dce23 100644 --- a/compose/2.10/src/ckanext-d4science/README.md +++ b/compose/2.10/src/ckanext-d4science/README.md @@ -20,6 +20,7 @@ Compatibility with core CKAN versions: | 2.7 | not tested | | 2.8 | not tested | | 2.9 | not tested | +| 2.10 | not tested | Suggested values: diff --git a/compose/2.10/src/ckanext-d4science/ckanext/d4science/tests/test_plugin.py b/compose/2.10/src/ckanext-d4science/ckanext/d4science/tests/test_plugin.py index ee3f39f..4313b47 100644 --- a/compose/2.10/src/ckanext-d4science/ckanext/d4science/tests/test_plugin.py +++ b/compose/2.10/src/ckanext-d4science/ckanext/d4science/tests/test_plugin.py @@ -3,7 +3,7 @@ Tests for plugin.py. Tests are written using the pytest library (https://docs.pytest.org), and you should read the testing guidelines in the CKAN docs: -https://docs.ckan.org/en/2.9/contributing/testing.html +https://docs.ckan.org/en/2.10/contributing/testing.html To write tests for your extension you should install the pytest-ckan package: diff --git a/images/ckan/2.10/Dockerfile.focal b/images/ckan/2.10/Dockerfile.focal new file mode 100644 index 0000000..db5aded --- /dev/null +++ b/images/ckan/2.10/Dockerfile.focal @@ -0,0 +1,253 @@ +################## +### Build CKAN ### +################## +FROM ubuntu:focal-20210827 as ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.10.4-focal + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.10.4 + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set Locale +ENV LC_ALL=en_US.UTF-8 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Set the locale +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Instal apt-utils +RUN apt-get install --no-install-recommends -y \ + apt-utils + +# Packages to build CKAN requirements and plugins +RUN apt-get install --no-install-recommends -y \ + git \ + curl \ + ca-certificates \ + python3 \ + libpq-dev \ + linux-headers-generic \ + gcc-10 \ + make \ + g++-10 \ + autoconf \ + automake \ + libtool \ + patch \ + libpcre3-dev \ + libpcre3 \ + python3-dev \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Use gcc 10 +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10 + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + gcc-aarch64-linux-gnu + +RUN apt-get update && apt-get install -y \ + libproj-dev \ + proj-data \ + proj-bin + +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 + +########################### +### Default-Extensions #### +########################### +FROM ubuntu:focal-20210827 as extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.1 + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + git \ + curl \ + ca-certificates \ + python3 \ + python3-dev + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM ubuntu:focal-20210827 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set Locale +ENV LC_ALL=en_US.UTF-8 + +# Set the locale +RUN apt-get update && \ + apt-get install --no-install-recommends -y locales && \ + sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=${LC_ALL} && \ + rm -rf /var/lib/apt/lists/* + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher + +# Install necessary packages to run CKAN +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + gettext \ + curl \ + ca-certificates \ + libpq5 \ + git \ + postgresql-client \ + python3 \ + python3-distutils \ + libpython3.8 \ + libmagic1 \ + libpcre3 \ + libxslt1.1 \ + libxml2 \ + tzdata \ + apache2-utils && \ + rm -rf /var/lib/apt/lists/* && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} && \ + # Link python to python3 + ln -s /usr/bin/python3 /usr/bin/python + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2 + +# Create a local user and group to run the app +RUN groupadd -g 92 ckan && \ + useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +ONBUILD RUN mkdir docker-afterinit.d + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] + diff --git a/src/ckanext-d4science/README.md b/src/ckanext-d4science/README.md index 17f8649..7683799 100644 --- a/src/ckanext-d4science/README.md +++ b/src/ckanext-d4science/README.md @@ -20,6 +20,7 @@ Compatibility with core CKAN versions: | 2.7 | not tested | | 2.8 | not tested | | 2.9 | not tested | +| 2.10 | not tested | Suggested values: diff --git a/src/ckanext-d4science/ckanext/d4science/tests/test_plugin.py b/src/ckanext-d4science/ckanext/d4science/tests/test_plugin.py index ee3f39f..4313b47 100644 --- a/src/ckanext-d4science/ckanext/d4science/tests/test_plugin.py +++ b/src/ckanext-d4science/ckanext/d4science/tests/test_plugin.py @@ -3,7 +3,7 @@ Tests for plugin.py. Tests are written using the pytest library (https://docs.pytest.org), and you should read the testing guidelines in the CKAN docs: -https://docs.ckan.org/en/2.9/contributing/testing.html +https://docs.ckan.org/en/2.10/contributing/testing.html To write tests for your extension you should install the pytest-ckan package: