From 63c484cfa965af750867b6735b22ff11a416ff2b Mon Sep 17 00:00:00 2001 From: amercader Date: Sun, 16 Feb 2020 22:27:02 +0100 Subject: [PATCH] Update Travis setup for py2/py3 testing --- .travis.yml | 33 ++++++++------ bin/travis-build.bash | 54 +++++++++++++++++----- bin/travis-run.bash | 17 +++++++ bin/travis-run.sh | 3 -- here_patch.diff | 40 +++++++++++++++++ test-core.ini | 70 ----------------------------- test-core-nose.ini => test-nose.ini | 0 test.ini | 26 ++++++++--- 8 files changed, 142 insertions(+), 101 deletions(-) create mode 100644 bin/travis-run.bash delete mode 100644 bin/travis-run.sh create mode 100644 here_patch.diff delete mode 100644 test-core.ini rename test-core-nose.ini => test-nose.ini (100%) diff --git a/.travis.yml b/.travis.yml index befc724..968dfa3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,9 @@ language: python -python: - - "2.7" -env: - - CKANVERSION=master - - CKANVERSION=2.8 - - CKANVERSION=2.7 - - CKANVERSION=2.6 services: - - redis-server + - redis - postgresql -install: - - bash bin/travis-build.bash -script: sh bin/travis-run.sh +install: bash bin/travis-build.bash +script: bash bin/travis-run.bash # the new trusty images of Travis cause build errors with psycopg2, see https://github.com/travis-ci/travis-ci/issues/8897 dist: trusty @@ -19,14 +11,14 @@ group: deprecated-2017Q4 stages: - Flake8 - - test + - Tests jobs: include: - stage: Flake8 + python: 2.7 env: FLAKE8=True install: - - bash bin/travis-build.bash - pip install flake8==3.5.0 - pip install pycodestyle==2.3.0 script: @@ -35,3 +27,18 @@ jobs: - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero + - stage: Tests + python: "2.7" + env: CKANVERSION=master + - python: "3.6" + env: CKANVERSION=master + - python: "2.7" + env: CKANVERSION=2.8 + - python: "2.7" + env: CKANVERSION=2.7 + - python: "2.7" + env: CKANVERSION=2.6 + +cache: + directories: + - $HOME/.cache/pip diff --git a/bin/travis-build.bash b/bin/travis-build.bash index 9f4ca66..b862876 100644 --- a/bin/travis-build.bash +++ b/bin/travis-build.bash @@ -2,10 +2,20 @@ set -e echo "This is travis-build.bash..." +echo "Targetting CKAN $CKANVERSION on Python $TRAVIS_PYTHON_VERSION" +if [ $CKANVERSION == 'master' ] +then + export CKAN_MINOR_VERSION=100 +else + export CKAN_MINOR_VERSION=${CKANVERSION##*.} +fi + +export PYTHON_MAJOR_VERSION=${TRAVIS_PYTHON_VERSION%.*} + echo "Installing the packages that CKAN requires..." sudo apt-get update -qq -sudo apt-get install solr-jetty libcommons-fileupload-java +sudo apt-get install solr-jetty echo "Installing CKAN and its Python dependencies..." git clone https://github.com/ckan/ckan @@ -18,14 +28,17 @@ else git checkout $CKAN_TAG echo "CKAN version: ${CKAN_TAG#ckan-}" fi + python setup.py develop -if [ -f requirements-py2.txt ] + +if (( $CKAN_MINOR_VERSION >= 9 )) && (( $PYTHON_MAJOR_VERSION == 2 )) then pip install -r requirements-py2.txt else pip install -r requirements.txt fi -pip install -r dev-requirements.txt --allow-all-external + +pip install -r dev-requirements.txt cd - echo "Setting up Solr..." @@ -45,20 +58,41 @@ sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default echo "Initialising the database..." cd ckan -paster db init -c test-core.ini +if (( $CKAN_MINOR_VERSION >= 9 )) +then + ckan -c test-core.ini db init +else + paster db init -c test-core.ini +fi cd - echo "Installing ckanext-harvest and its requirements..." -pip install -r pip-requirements.txt --allow-all-external -pip install -r dev-requirements.txt --allow-all-external +pip install -r pip-requirements.txt +pip install -r dev-requirements.txt python setup.py develop -paster harvester initdb -c ckan/test-core.ini -echo "Moving test.ini into a subdir..." +if (( $CKAN_MINOR_VERSION >= 9 )) +then +echo "Patching CKAN until #5204 is fixed" + cd ckan + patch -p1 < ../here_patch.diff + cd - +fi + +echo "Moving test.ini into a subdir... (because the core ini file is referenced as ../ckan/test-core.ini)" mkdir subdir -mv test-core.ini subdir -mv test-core-nose.ini subdir +mv test.ini subdir +mv test-nose.ini subdir + + +if (( $CKAN_MINOR_VERSION >= 9 )) +then + ckan -c subdir/test.ini harvester initdb +else + paster harvester initdb -c subdir/test.ini +fi + echo "travis-build.bash is done." diff --git a/bin/travis-run.bash b/bin/travis-run.bash new file mode 100644 index 0000000..267c9e5 --- /dev/null +++ b/bin/travis-run.bash @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [ $CKANVERSION == 'master' ] +then + export CKAN_MINOR_VERSION=100 +else + export CKAN_MINOR_VERSION=${CKANVERSION##*.} +fi + + +if (( $CKAN_MINOR_VERSION >= 9 )) +then + pytest --ckan-ini=subdir/test.ini --cov=ckanext.harvest ckanext/harvest/tests +else + nosetests --ckan --nologcapture --with-pylons=subdir/test-nose.ini --with-coverage --cover-package=ckanext.harvest --cover-inclusive --cover-erase --cover-tests ckanext/harvest/tests/nose +fi diff --git a/bin/travis-run.sh b/bin/travis-run.sh deleted file mode 100644 index 07b727f..0000000 --- a/bin/travis-run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -e - -nosetests --ckan --nologcapture --with-pylons=subdir/test-core-nose.ini -v ckanext/harvest/tests/nose diff --git a/here_patch.diff b/here_patch.diff new file mode 100644 index 0000000..7bd0bf4 --- /dev/null +++ b/here_patch.diff @@ -0,0 +1,40 @@ +diff --git a/ckan/cli/__init__.py b/ckan/cli/__init__.py +index 490021276..6053d4601 100644 +--- a/ckan/cli/__init__.py ++++ b/ckan/cli/__init__.py +@@ -17,12 +17,6 @@ class CKANConfigLoader(object): + self.section = u'app:main' + self.read_config_files(filename) + +- defaults = { +- u'here': os.path.dirname(os.path.abspath(filename)), +- u'__file__': os.path.abspath(filename) +- } +- self._update_defaults(defaults) +- + def read_config_files(self, filename): + '''Read and parses a config file. + +@@ -33,12 +27,22 @@ class CKANConfigLoader(object): + configs in extension's test.ini files will override the configs of + test-core.ini. + ''' ++ defaults = { ++ u'here': os.path.dirname(os.path.abspath(filename)), ++ u'__file__': os.path.abspath(filename) ++ } ++ self._update_defaults(defaults) + self.parser.read(filename) + + schema, path = self.parser.get(self.section, u'use').split(u':') + if schema == u'config': + path = os.path.join( + os.path.dirname(os.path.abspath(filename)), path) ++ ++ defaults = { ++ u'here': os.path.dirname(os.path.abspath(path)), ++ } ++ self._update_defaults(defaults) + self.parser.read([path, filename]) + + def _update_defaults(self, new_defaults): diff --git a/test-core.ini b/test-core.ini deleted file mode 100644 index ffbd4f5..0000000 --- a/test-core.ini +++ /dev/null @@ -1,70 +0,0 @@ -[DEFAULT] -debug = false -# Uncomment and replace with the address which should receive any error reports -#email_to = you@yourdomain.com -smtp_server = localhost -error_email_from = paste@localhost - -[server:main] -use = egg:Paste#http -host = 0.0.0.0 -port = 5000 - - -[app:main] -use = config:../ckan/test-core.ini -# Here we hard-code the database and a flag to make default tests -# run fast. -ckan.plugins = harvest ckan_harvester test_harvester test_harvester2 test_action_harvester -ckan.harvest.mq.type = redis -ckan.legacy_templates = false -# NB: other test configuration should go in test-core.ini, which is -# what the postgres tests use. - - -# Logging configuration -[loggers] -keys = root, ckan, sqlalchemy - -[handlers] -keys = console, dblog - -[formatters] -keys = generic, dblog - -[logger_root] -level = WARN -handlers = console - -[logger_ckan] -qualname = ckan -handlers = -level = INFO - -[logger_ckan_harvester] -qualname = ckanext.harvest -handlers = dblog -level = DEBUG - -[logger_sqlalchemy] -handlers = -qualname = sqlalchemy.engine -level = WARN - -[handler_console] -class = StreamHandler -args = (sys.stdout,) -level = NOTSET -formatter = generic - -[handler_dblog] -class = ckanext.harvest.log.DBLogHandler -args = () -level = DEBUG -formatter = dblog - -[formatter_dblog] -format = %(message)s - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/test-core-nose.ini b/test-nose.ini similarity index 100% rename from test-core-nose.ini rename to test-nose.ini diff --git a/test.ini b/test.ini index c28f069..ffbd4f5 100644 --- a/test.ini +++ b/test.ini @@ -1,5 +1,5 @@ [DEFAULT] -debug = true +debug = false # Uncomment and replace with the address which should receive any error reports #email_to = you@yourdomain.com smtp_server = localhost @@ -12,10 +12,12 @@ port = 5000 [app:main] -use = config:../ckan/test.ini +use = config:../ckan/test-core.ini # Here we hard-code the database and a flag to make default tests # run fast. -ckan.plugins = harvest ckan_harvester test_harvester +ckan.plugins = harvest ckan_harvester test_harvester test_harvester2 test_action_harvester +ckan.harvest.mq.type = redis +ckan.legacy_templates = false # NB: other test configuration should go in test-core.ini, which is # what the postgres tests use. @@ -25,10 +27,10 @@ ckan.plugins = harvest ckan_harvester test_harvester keys = root, ckan, sqlalchemy [handlers] -keys = console +keys = console, dblog [formatters] -keys = generic +keys = generic, dblog [logger_root] level = WARN @@ -39,6 +41,11 @@ qualname = ckan handlers = level = INFO +[logger_ckan_harvester] +qualname = ckanext.harvest +handlers = dblog +level = DEBUG + [logger_sqlalchemy] handlers = qualname = sqlalchemy.engine @@ -50,5 +57,14 @@ args = (sys.stdout,) level = NOTSET formatter = generic +[handler_dblog] +class = ckanext.harvest.log.DBLogHandler +args = () +level = DEBUG +formatter = dblog + +[formatter_dblog] +format = %(message)s + [formatter_generic] format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s