From f12464972c5e6c9176a91f41b094f7b1ce188fa0 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 Aug 2022 13:00:32 +0200 Subject: [PATCH] Separate test workflows for postgis and default setup --- .github/workflows/test-postgis.yml | 104 +++++++++++++++++++++++++++++ .github/workflows/test.yml | 13 +--- requirements-postgis.txt | 1 + requirements-py2.txt | 2 +- requirements.txt | 1 - test-postgis.ini | 50 ++++++++++++++ 6 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/test-postgis.yml create mode 100644 requirements-postgis.txt create mode 100644 test-postgis.ini diff --git a/.github/workflows/test-postgis.yml b/.github/workflows/test-postgis.yml new file mode 100644 index 0000000..1de5ee2 --- /dev/null +++ b/.github/workflows/test-postgis.yml @@ -0,0 +1,104 @@ +name: Tests +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.6' + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + test: + needs: lint + strategy: + matrix: + ckan-version: [master, 2.9, 2.9-py2, 2.8, 2.7] + fail-fast: false + + name: CKAN ${{ matrix.ckan-version }} + runs-on: ubuntu-latest + container: + image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + services: + solr: + image: ckan/ckan-solr-dev:${{ matrix.ckan-version }} + postgres: + image: postgis/postgis:10-3.1 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:3 + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 + PGPASSWORD: postgres + + steps: + - uses: actions/checkout@v2 + - name: Create Database + run: | + psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" + createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test + psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" + psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" + createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test + - name: Install harvester + run: | + git clone https://github.com/ckan/ckanext-harvest + cd ckanext-harvest + pip install -r pip-requirements.txt + pip install -r dev-requirements.txt + pip install -e . + - name: Install dependency (common) + run: | + apk add --no-cache \ + geos \ + geos-dev \ + proj-util \ + proj-dev \ + libxml2 \ + libxslt \ + gcc \ + libxml2-dev \ + libxslt-dev + - name: Install dependency (python2) + if: ${{ matrix.ckan-version == '2.9-py2' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.7' }} + run: | + apk add --no-cache \ + python2-dev + pip install -r requirements-py2.txt + - name: Install dependency (python3) + if: ${{ matrix.ckan-version != '2.9-py2' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.7' }} + run: | + apk add --no-cache \ + python3-dev + pip install -r requirements.txt + - name: Install requirements + run: | + + pip install -r requirements-postgis.txt + pip install -e . + # Replace default path to CKAN core config file with the one on the container + sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: setup postgis + run: | + psql --host=postgres --username=postgres -d ckan_test --command="ALTER ROLE ckan_default WITH superuser;" + psql --host=postgres --username=postgres -d ckan_test --command="CREATE EXTENSION postgis;" + - name: Run tests + run: pytest --ckan-ini=test-postgis.ini --cov=ckanext.spatial --cov-report=xml --cov-append --disable-warnings ckanext/spatial/tests + + - name: Upload coverage report to codecov + uses: codecov/codecov-action@v1 + with: + file: ./coverage.xml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d12e7f4..81476f3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: solr: image: ckan/ckan-solr-dev:${{ matrix.ckan-version }} postgres: - image: postgis/postgis:10-3.1 + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -46,13 +46,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Create Database - run: | - psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" - createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test - psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" - psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" - createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test - name: Install harvester run: | git clone https://github.com/ckan/ckanext-harvest @@ -89,10 +82,6 @@ jobs: pip install -e . # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini - - name: setup postgis - run: | - psql --host=postgres --username=postgres -d ckan_test --command="ALTER ROLE ckan_default WITH superuser;" - psql --host=postgres --username=postgres -d ckan_test --command="CREATE EXTENSION postgis;" - name: Run tests run: pytest --ckan-ini=test.ini --cov=ckanext.spatial --cov-report=xml --cov-append --disable-warnings ckanext/spatial/tests diff --git a/requirements-postgis.txt b/requirements-postgis.txt new file mode 100644 index 0000000..1f1cb0d --- /dev/null +++ b/requirements-postgis.txt @@ -0,0 +1 @@ +GeoAlchemy2==0.12.3 diff --git a/requirements-py2.txt b/requirements-py2.txt index a748016..660d059 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -1,5 +1,4 @@ ckantoolkit -GeoAlchemy2==0.12.3 Shapely>=1.2.13 pyproj==2.2.2 OWSLib==0.18.0 @@ -8,3 +7,4 @@ argparse pyparsing>=2.1.10 requests>=1.1.0 six +geojson==2.5.0 diff --git a/requirements.txt b/requirements.txt index 482b723..315b067 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ ckantoolkit -GeoAlchemy2==0.12.3 Shapely>=1.2.13 pyproj==2.6.1 OWSLib==0.18.0 diff --git a/test-postgis.ini b/test-postgis.ini new file mode 100644 index 0000000..3e84d55 --- /dev/null +++ b/test-postgis.ini @@ -0,0 +1,50 @@ +[DEFAULT] +debug = false +# Uncomment and replace with the address which should receive any error reports +#email_to = you@yourdomain.com +smtp_server = localhost +error_email_from = paste@localhost + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + + +[app:main] +use = config:test.ini +ckanext.spatial.search_backend = postgis +ckan.spatial.use_postgis = true + +# Logging configuration +[loggers] +keys = root, ckan, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_ckan] +qualname = ckan +handlers = +level = INFO + +[logger_sqlalchemy] +handlers = +qualname = sqlalchemy.engine +level = WARN + +[handler_console] +class = StreamHandler +args = (sys.stdout,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s