More updates

datapusher/Dockerfile

@@ -1,37 +1,53 @@
-FROM alpine:3.7
+FROM alpine:3.13
 
 ENV APP_DIR=/srv/app
-ENV GIT_BRANCH master
+ENV GIT_BRANCH 0.0.17
 ENV GIT_URL https://github.com/ckan/datapusher.git
 ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py
 
 WORKDIR ${APP_DIR}
 
-RUN apk add --no-cache python \
-    py-pip \
-    py-gunicorn \
+RUN apk add --no-cache \
+    python3 \
+    py3-pip \
+    py3-wheel \
     libffi-dev \
     libressl-dev \
-    libxslt && \
-    # Temporary packages to build CKAN requirements
-    apk add --no-cache --virtual .build-deps \
+    libxslt \
+    uwsgi \
+    uwsgi-http \
+    uwsgi-corerouter \
+    uwsgi-python \
+    # Temporary packages to build DataPusher requirements
+    && apk add --no-cache --virtual .build-deps \
     gcc \
     git \
     musl-dev \
-    python-dev \
+    python3-dev \
     libxml2-dev \
-    libxslt-dev && \
-    # Fetch datapusher and install
-    mkdir ${APP_DIR}/src && cd ${APP_DIR}/src && \
+    libxslt-dev \
+    libmagic \
+    openssl-dev \
+    cargo
+
+RUN mkdir ${APP_DIR}/src && cd ${APP_DIR}/src && \
     git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \
     cd datapusher && \
-    python setup.py install && \
-    pip install --no-cache-dir -r requirements.txt && \
-    # Remove temporary packages and files
-    apk del .build-deps && \
+    python3 setup.py install && \
+    pip3 install --no-cache-dir -r requirements.txt
+
+RUN apk del .build-deps && \
+    cp ${APP_DIR}/src/datapusher/deployment/*.* ${APP_DIR} && \
+    # Remove default values in ini file
+    sed -i '/http/d' ${APP_DIR}/datapusher-uwsgi.ini && \
+    sed -i '/wsgi-file/d' ${APP_DIR}/datapusher-uwsgi.ini && \
+    sed -i '/virtualenv/d' ${APP_DIR}/datapusher-uwsgi.ini && \
     rm -rf ${APP_DIR}/src
 
-COPY setup ${APP_DIR}
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S www-data && \
+    adduser -u 92 -h /srv/app -H -D -S -G www-data www-data    
 
 EXPOSE 8800
-CMD ["gunicorn", "--bind=0.0.0.0:8800", "--log-file=-", "wsgi"]
+CMD ["sh", "-c", \
+    "uwsgi --plugins=http,python --http=0.0.0.0:8800 --socket=/tmp/uwsgi.sock --ini=`echo ${APP_DIR}`/datapusher-uwsgi.ini --wsgi-file=`echo ${APP_DIR}`/datapusher.wsgi"]

datapusher/setup/datapusher_settings.py

@@ -1,29 +0,0 @@
-import uuid
-
-DEBUG = False
-TESTING = False
-SECRET_KEY = str(uuid.uuid4())
-USERNAME = str(uuid.uuid4())
-PASSWORD = str(uuid.uuid4())
-
-NAME = 'datapusher'
-
-# database
-
-SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/job_store.db'
-
-# webserver host and port
-
-HOST = '0.0.0.0'
-PORT = 8800
-
-# logging
-
-#FROM_EMAIL = 'server-error@example.com'
-#ADMINS = ['yourname@example.com']  # where to send emails
-
-#LOG_FILE = '/tmp/ckan_service.log'
-STDERR = True
-
-# cloud settings
-MAX_CONTENT_LENGTH = 73400320

datapusher/setup/wsgi.py

@@ -1,9 +0,0 @@
-import os
-import sys
-
-import ckanserviceprovider.web as web
-web.init()
-
-import datapusher.jobs as jobs
-
-application = web.app

docker-compose.yml

@@ -48,6 +48,7 @@ services:
     environment:
       - DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - PGDATA=/var/lib/postgresql/data/db
     volumes:
       - pg_data:/var/lib/postgresql/data