Separate test workflows for postgis and default setup

2022-08-24 13:00:32 +02:00 · 2022-08-24 13:00:32 +02:00 · f12464972c
parent 6f8870ee58
commit f12464972c
6 changed files with 157 additions and 14 deletions
--- a/.github/workflows/test-postgis.yml
+++ b/.github/workflows/test-postgis.yml
@ -0,0 +1,104 @@
+name: Tests
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        ckan-version: [master, 2.9, 2.9-py2, 2.8, 2.7]
+      fail-fast: false
+    
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+      postgres:
+        image: postgis/postgis:10-3.1
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+      PGPASSWORD: postgres
+    
+    steps:
+    - uses: actions/checkout@v2
+    - name: Create Database
+      run: |
+        psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test
+        psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test
+    - name: Install harvester
+      run: |
+        git clone https://github.com/ckan/ckanext-harvest
+        cd ckanext-harvest
+        pip install -r pip-requirements.txt
+        pip install -r dev-requirements.txt
+        pip install -e .
+    - name: Install dependency (common)
+      run: |
+        apk add --no-cache \
+          geos \
+          geos-dev \
+          proj-util \
+          proj-dev \
+          libxml2 \
+          libxslt \
+          gcc \
+          libxml2-dev \
+          libxslt-dev
+    - name: Install dependency (python2)
+      if: ${{ matrix.ckan-version == '2.9-py2' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.7' }}
+      run: |
+        apk add --no-cache \
+          python2-dev
+        pip install -r requirements-py2.txt
+    - name: Install dependency (python3)
+      if: ${{ matrix.ckan-version != '2.9-py2' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.7' }}
+      run: |
+        apk add --no-cache \
+          python3-dev
+        pip install -r requirements.txt
+    - name: Install requirements
+      run: |
+
+        pip install -r requirements-postgis.txt
+        pip install -e .
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+    - name: setup postgis
+      run: |
+        psql --host=postgres --username=postgres -d ckan_test --command="ALTER ROLE ckan_default WITH superuser;"
+        psql --host=postgres --username=postgres -d ckan_test --command="CREATE EXTENSION postgis;"
+    - name: Run tests
+      run: pytest --ckan-ini=test-postgis.ini --cov=ckanext.spatial --cov-report=xml --cov-append --disable-warnings ckanext/spatial/tests
+
+    - name: Upload coverage report to codecov
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.xml
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -28,7 +28,7 @@ jobs:
      solr:
        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
      postgres:
-        image: postgis/postgis:10-3.1
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
@ -46,13 +46,6 @@ jobs:
    
    steps:
    - uses: actions/checkout@v2
-    - name: Create Database
-      run: |
-        psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test
-        psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-        psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test
    - name: Install harvester
      run: |
        git clone https://github.com/ckan/ckanext-harvest
@ -89,10 +82,6 @@ jobs:
        pip install -e .
        # Replace default path to CKAN core config file with the one on the container
        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
-    - name: setup postgis
-      run: |
-        psql --host=postgres --username=postgres -d ckan_test --command="ALTER ROLE ckan_default WITH superuser;"
-        psql --host=postgres --username=postgres -d ckan_test --command="CREATE EXTENSION postgis;"
    - name: Run tests
      run: pytest --ckan-ini=test.ini --cov=ckanext.spatial --cov-report=xml --cov-append --disable-warnings ckanext/spatial/tests

--- a/requirements-postgis.txt
+++ b/requirements-postgis.txt
@ -0,0 +1 @@
+GeoAlchemy2==0.12.3
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@ -1,5 +1,4 @@
 ckantoolkit
-GeoAlchemy2==0.12.3
 Shapely>=1.2.13
 pyproj==2.2.2
 OWSLib==0.18.0
@ -8,3 +7,4 @@ argparse
 pyparsing>=2.1.10
 requests>=1.1.0
 six
+geojson==2.5.0
--- a/requirements.txt
+++ b/requirements.txt
@ -1,5 +1,4 @@
 ckantoolkit
-GeoAlchemy2==0.12.3
 Shapely>=1.2.13
 pyproj==2.6.1
 OWSLib==0.18.0
--- a/test-postgis.ini
+++ b/test-postgis.ini
@ -0,0 +1,50 @@
+[DEFAULT]
+debug = false
+# Uncomment and replace with the address which should receive any error reports
+#email_to = you@yourdomain.com
+smtp_server = localhost
+error_email_from = paste@localhost
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+
+[app:main]
+use = config:test.ini
+ckanext.spatial.search_backend = postgis
+ckan.spatial.use_postgis = true
+
+# Logging configuration
+[loggers]
+keys = root, ckan, sqlalchemy
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_ckan]
+qualname = ckan
+handlers =
+level = INFO
+
+[logger_sqlalchemy]
+handlers =
+qualname = sqlalchemy.engine
+level = WARN
+
+[handler_console]
+class = StreamHandler
+args = (sys.stdout,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s