Create new tag with xloader as default plugin, replacement for datapusher

.github/workflows/master_merge.yml

@@ -54,6 +54,23 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
 
+      - name: Get docker tag for alpine xloader image
+        id: alpine
+        run: |
+          echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT
+
+      - name: Build and push CKAN 2.10 alpine xloader
+        uses: docker/build-push-action@v5
+        with:
+          context: ./images/ckan/2.10
+          file: ./images/ckan/2.10/Dockerfile.xloader
+          push: true
+          tags: |
+            keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+            ghcr.io/keitaroinc/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+          cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
+
       - name: Get docker tag for Ubuntu image
         id: ubuntu
         run: |

.github/workflows/pr_checks.yml

@@ -39,6 +39,21 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
 
+      - name: Get docker tag for Alpine xloader image
+        id: alpine
+        run: |
+          echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT
+
+      - name: Build CKAN 2.10 alpine
+        uses: docker/build-push-action@v5
+        with:
+          context: ./images/ckan/2.10
+          file: ./images/ckan/2.10/Dockerfile.xloader
+          push: false
+          tags: keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+          cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
+
       - name: Get docker tag for Ubuntu image
         id: ubuntu
         run: |

.github/workflows/trivy_scan.yml

@@ -55,6 +55,30 @@ jobs:
         with:
           sarif_file: 'trivy-results.sarif'
 
+  scan_2_10_xloader:
+    name: scan_2_10_xloader
+    runs-on: ubuntu-20.04
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build an image from Dockerfile
+        run: |
+          docker build -t keitaro/ckan/2.10:${{ github.sha }} -f ./images/ckan/2.10/Dockerfile.xloader ./images/ckan/2.10/
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'keitaro/ckan/2.10:${{ github.sha }}'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: 'trivy-results.sarif'
+
 
   scan_2_10_focal:
     name: scan_2_10_focal

images/ckan/2.10/Dockerfile.xloader

@@ -0,0 +1,207 @@
+##################
+### Build CKAN ###
+##################
+FROM alpine:3.17.2 as ckanbuild
+
+# Used by Github Actions to tag the image with
+ENV IMAGE_TAG=2.10.4
+
+# Set CKAN version to build
+ENV GIT_URL=https://github.com/ckan/ckan.git
+ENV GIT_BRANCH=ckan-2.10.4
+
+# Set src dirs
+ENV SRC_DIR=/srv/app/src
+ENV PIP_SRC=${SRC_DIR}
+
+WORKDIR ${SRC_DIR}
+
+# Packages to build CKAN requirements and plugins
+RUN apk add --no-cache \
+        python3 \
+        python3-dev \
+        git \
+        curl \
+        postgresql-dev \
+        linux-headers \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+        libtool \
+        patch \
+        musl-dev \
+        pcre-dev \
+        pcre \
+        libffi-dev \
+        libxml2-dev \
+        libxslt-dev
+
+# Create the src directory
+RUN mkdir -p ${SRC_DIR}
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py
+    
+# Downgrade setuptools so that CKAN requirements can be built
+RUN pip install setuptools==44.1.0
+
+# Fetch and build CKAN and requirements
+RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+# RUN ${SRC_DIR}/apply_ckan_patches.sh
+RUN rm -rf /srv/app/src/ckan/.git
+RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
+RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2
+
+
+###########################
+### Default-Extensions ####
+###########################
+FROM alpine:3.17.2 as extbuild
+
+# Set src dirs
+ENV SRC_DIR=/srv/app/src
+ENV PIP_SRC=${SRC_DIR}
+
+# List of default extensions
+ENV DEFAULT_EXTENSIONS envvars
+
+# Locations and tags, please use specific tags or revisions
+ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
+ENV ENVVARS_GIT_BRANCH=0.0.2
+
+RUN apk add --no-cache \
+        python3 \
+        python3-dev \
+        git \
+        curl 
+
+# Create the src directory
+RUN mkdir -p ${SRC_DIR}
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py
+
+# Fetch and build the default CKAN extensions
+RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
+
+############
+### MAIN ###
+############
+FROM alpine:3.17.2
+
+LABEL maintainer="Keitaro Inc <info@keitaro.com>"
+LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
+
+ENV APP_DIR=/srv/app
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_DIR=${SRC_DIR}/ckan
+ENV DATA_DIR=/srv/app/data
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore xloader
+
+# Install necessary packages to run CKAN
+RUN apk add --no-cache \
+        supervisor \
+        python3 \
+        bash \
+        git \
+        gettext \
+        curl \
+        postgresql-client \
+        libmagic \
+        pcre \
+        libxslt \
+        libxml2 \
+        tzdata \
+        apache2-utils && \
+    # Create SRC_DIR
+    mkdir -p ${SRC_DIR}
+
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py
+
+    ### XLoader ###
+RUN pip3 install -e 'git+https://github.com/keitaroinc/ckanext-xloader.git@master#egg=ckanext-xloader' && \ 
+   pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt && \
+   pip3 install -U requests[security]
+
+# Get artifacts from build stages
+COPY --from=ckanbuild /wheels /srv/app/wheels
+COPY --from=extbuild /wheels /srv/app/ext_wheels
+COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
+
+# Additional install steps for build stages artifacts
+RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+WORKDIR ${CKAN_DIR}
+
+# Install CKAN
+RUN pip install -e /srv/app/src/ckan && \
+    cp who.ini ${APP_DIR} && \
+    pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
+    # Install default CKAN extensions
+    pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
+    # Create and update CKAN config
+    # Set timezone
+    echo "UTC" >  /etc/timezone && \
+    # Generate CKAN config
+    ckan generate config ${APP_DIR}/production.ini && \
+    ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
+    # Configure plugins
+    ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
+    # Create the data directory
+    mkdir ${DATA_DIR} && \
+    # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created
+    ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \
+    # Set the default level for extensions to INFO
+    ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
+    # Change ownership to app user
+    chown -R ckan:ckan /srv/app
+
+# Remove wheels
+RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
+
+# Copy necessary scripts
+COPY setup/app ${APP_DIR}
+
+WORKDIR ${APP_DIR}
+
+# Create entrypoint directory for children image scripts
+RUN mkdir docker-entrypoint.d
+
+# Create afterinit directory for children image scripts
+RUN mkdir docker-afterinit.d
+
+RUN pip install supervisor
+RUN mkdir /var/supervisor
+RUN mkdir /var/log/ckan
+RUN mkdir /etc/supervisor/conf.d/ -p
+COPY supervisorvalidation.conf /etc/supervisor/conf.d/supervisor-ckan-worker.conf
+COPY supervisord.conf /etc/supervisord.conf
+COPY supervisorstart.sh /srv/app/docker-afterinit.d
+
+# Change ownership to app user
+RUN chown -R ckan:ckan ${APP_DIR} /var/supervisor /var/log/ckan /etc/supervisor/conf.d/
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+USER ckan
+
+CMD ["/srv/app/start_ckan.sh"]

images/ckan/2.10/supervisord.conf

@@ -0,0 +1,23 @@
+; supervisor config file
+[unix_http_server]
+file=/var/supervisor/supervisor.sock   ; (the path to the socket file)
+chmod=0700                       ; sockef file mode (default 0700)
+[supervisord]
+logfile=/var/log/ckan/supervisord.log ; (main log file;default $CWD/supervisord.log)
+;pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+;childlogdir=/var/log/supervisor            ; ('AUTO' child log dir, default $TEMP)
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+user=ckan
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+[supervisorctl]
+serverurl=unix:///var/supervisor/supervisor.sock ; use a unix:// URL  for a unix socket
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+[include]
+files = /etc/supervisor/conf.d/*.conf

images/ckan/2.10/supervisorstart.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+supervisord

images/ckan/2.10/supervisorvalidation.conf

@@ -0,0 +1,54 @@
+; =======================================================
+; Supervisor configuration for CKAN background job worker
+; =======================================================
+; 1. Copy this file to /etc/supervisor/conf.d
+; 2. Make sure the paths below match your setup
+[program:ckan-worker-default]
+command=ckan -c /srv/app/production.ini jobs worker default
+; Log files.
+redirect_stderr=true
+stdout_logfile=/var/log/ckan/ckan-worker.log
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
+
+[program:ckan-worker-bulk]
+command=ckan -c /srv/app/production.ini jobs worker bulk
+; Log files.
+redirect_stderr=true
+stdout_logfile=/var/log/ckan/ckan-worker.log
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
+
+[program:ckan-worker-priority]
+command=ckan -c /srv/app/production.ini jobs worker priority
+numprocs=2
+process_name=%(program_name)s-%(process_num)02d
+; Log files.
+redirect_stderr=true
+stdout_logfile=/var/log/ckan/ckan-worker.log
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600