Update Travis setup for py2/py3 testing

2020-02-16 22:27:02 +01:00 · 2020-02-16 22:27:02 +01:00 · 63c484cfa9
parent d2d5c34395
commit 63c484cfa9
8 changed files with 142 additions and 101 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,17 +1,9 @@
 language: python
-python:
-    - "2.7"
-env:
-    - CKANVERSION=master
-    - CKANVERSION=2.8
-    - CKANVERSION=2.7
-    - CKANVERSION=2.6
 services:
-    - redis-server
+    - redis
    - postgresql
-install:
-    - bash bin/travis-build.bash
-script: sh bin/travis-run.sh
+install: bash bin/travis-build.bash
+script: bash bin/travis-run.bash

 # the new trusty images of Travis cause build errors with psycopg2, see https://github.com/travis-ci/travis-ci/issues/8897
 dist: trusty
@ -19,14 +11,14 @@ group: deprecated-2017Q4

 stages:
  - Flake8
-  - test
+  - Tests

 jobs:
  include:
    - stage: Flake8
+      python: 2.7
      env: FLAKE8=True
      install:
-        - bash bin/travis-build.bash
        - pip install flake8==3.5.0
        - pip install pycodestyle==2.3.0
      script:
@ -35,3 +27,18 @@ jobs:
        - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
        # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
        - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero
+    - stage: Tests
+      python: "2.7"
+      env: CKANVERSION=master
+    - python: "3.6"
+      env: CKANVERSION=master
+    - python: "2.7"
+      env: CKANVERSION=2.8
+    - python: "2.7"
+      env: CKANVERSION=2.7
+    - python: "2.7"
+      env: CKANVERSION=2.6
+
+cache:
+  directories:
+    - $HOME/.cache/pip
--- a/bin/travis-build.bash
+++ b/bin/travis-build.bash
@ -2,10 +2,20 @@
 set -e

 echo "This is travis-build.bash..."
+echo "Targetting CKAN $CKANVERSION on Python $TRAVIS_PYTHON_VERSION"
+if [ $CKANVERSION == 'master' ]
+then
+    export CKAN_MINOR_VERSION=100
+else
+    export CKAN_MINOR_VERSION=${CKANVERSION##*.}
+fi
+
+export PYTHON_MAJOR_VERSION=${TRAVIS_PYTHON_VERSION%.*}
+

 echo "Installing the packages that CKAN requires..."
 sudo apt-get update -qq
-sudo apt-get install solr-jetty libcommons-fileupload-java
+sudo apt-get install solr-jetty

 echo "Installing CKAN and its Python dependencies..."
 git clone https://github.com/ckan/ckan
@ -18,14 +28,17 @@ else
    git checkout $CKAN_TAG
    echo "CKAN version: ${CKAN_TAG#ckan-}"
 fi
+
 python setup.py develop
-if [ -f requirements-py2.txt ]
+
+if (( $CKAN_MINOR_VERSION >= 9 )) && (( $PYTHON_MAJOR_VERSION == 2 ))
 then
    pip install -r requirements-py2.txt
 else
    pip install -r requirements.txt
 fi
-pip install -r dev-requirements.txt --allow-all-external
+
+pip install -r dev-requirements.txt
 cd -

 echo "Setting up Solr..."
@ -45,20 +58,41 @@ sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default

 echo "Initialising the database..."
 cd ckan
-paster db init -c test-core.ini
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+    ckan -c test-core.ini db init
+else
+    paster db init -c test-core.ini
+fi
 cd -

 echo "Installing ckanext-harvest and its requirements..."
-pip install -r pip-requirements.txt --allow-all-external
-pip install -r dev-requirements.txt --allow-all-external
+pip install -r pip-requirements.txt
+pip install -r dev-requirements.txt

 python setup.py develop

-paster harvester initdb -c ckan/test-core.ini

-echo "Moving test.ini into a subdir..."
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+echo "Patching CKAN until #5204 is fixed"
+    cd ckan
+    patch -p1 < ../here_patch.diff
+    cd -
+fi
+
+echo "Moving test.ini into a subdir... (because the core ini file is referenced as ../ckan/test-core.ini)"
 mkdir subdir
-mv test-core.ini subdir
-mv test-core-nose.ini subdir
+mv test.ini subdir
+mv test-nose.ini subdir
+
+
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+    ckan -c subdir/test.ini harvester initdb
+else
+    paster harvester initdb -c subdir/test.ini
+fi
+

 echo "travis-build.bash is done."
--- a/bin/travis-run.bash
+++ b/bin/travis-run.bash
@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+if [ $CKANVERSION == 'master' ]
+then
+    export CKAN_MINOR_VERSION=100
+else
+    export CKAN_MINOR_VERSION=${CKANVERSION##*.}
+fi
+
+
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+    pytest --ckan-ini=subdir/test.ini --cov=ckanext.harvest ckanext/harvest/tests
+else
+    nosetests --ckan --nologcapture --with-pylons=subdir/test-nose.ini --with-coverage --cover-package=ckanext.harvest --cover-inclusive --cover-erase --cover-tests ckanext/harvest/tests/nose
+fi
--- a/bin/travis-run.sh
+++ b/bin/travis-run.sh
@ -1,3 +0,0 @@
-#!/bin/sh -e
-
-nosetests --ckan --nologcapture --with-pylons=subdir/test-core-nose.ini -v ckanext/harvest/tests/nose
--- a/here_patch.diff
+++ b/here_patch.diff
@ -0,0 +1,40 @@
+diff --git a/ckan/cli/__init__.py b/ckan/cli/__init__.py
+index 490021276..6053d4601 100644
+--- a/ckan/cli/__init__.py
+++ b/ckan/cli/__init__.py
+@@ -17,12 +17,6 @@ class CKANConfigLoader(object):
+         self.section = u'app:main'
+         self.read_config_files(filename)
+ 
+-        defaults = {
+-            u'here': os.path.dirname(os.path.abspath(filename)),
+-            u'__file__': os.path.abspath(filename)
+-        }
+-        self._update_defaults(defaults)
+-
+     def read_config_files(self, filename):
+         '''Read and parses a config file.
+ 
+@@ -33,12 +27,22 @@ class CKANConfigLoader(object):
+         configs in extension's test.ini files will override the configs of
+         test-core.ini.
+         '''
+        defaults = {
+            u'here': os.path.dirname(os.path.abspath(filename)),
+            u'__file__': os.path.abspath(filename)
+        }
+        self._update_defaults(defaults)
+         self.parser.read(filename)
+ 
+         schema, path = self.parser.get(self.section, u'use').split(u':')
+         if schema == u'config':
+             path = os.path.join(
+                 os.path.dirname(os.path.abspath(filename)), path)
+
+            defaults = {
+                u'here': os.path.dirname(os.path.abspath(path)),
+            }
+            self._update_defaults(defaults)
+             self.parser.read([path, filename])
+ 
+     def _update_defaults(self, new_defaults):
--- a/test-core.ini
+++ b/test-core.ini
@ -1,70 +0,0 @@
-[DEFAULT]
-debug = false
-# Uncomment and replace with the address which should receive any error reports
-#email_to = you@yourdomain.com
-smtp_server = localhost
-error_email_from = paste@localhost
-
-[server:main]
-use = egg:Paste#http
-host = 0.0.0.0
-port = 5000
-
-
-[app:main]
-use = config:../ckan/test-core.ini
-# Here we hard-code the database and a flag to make default tests
-# run fast.
-ckan.plugins = harvest ckan_harvester test_harvester test_harvester2 test_action_harvester
-ckan.harvest.mq.type = redis
-ckan.legacy_templates = false
-# NB: other test configuration should go in test-core.ini, which is
-#     what the postgres tests use.
-
-
-# Logging configuration
-[loggers]
-keys = root, ckan, sqlalchemy
-
-[handlers]
-keys = console, dblog
-
-[formatters]
-keys = generic, dblog
-
-[logger_root]
-level = WARN
-handlers = console
-
-[logger_ckan]
-qualname = ckan
-handlers = 
-level = INFO
-
-[logger_ckan_harvester]
-qualname = ckanext.harvest
-handlers = dblog
-level = DEBUG
-
-[logger_sqlalchemy]
-handlers =
-qualname = sqlalchemy.engine
-level = WARN  
-
-[handler_console]
-class = StreamHandler
-args = (sys.stdout,)
-level = NOTSET
-formatter = generic
-
-[handler_dblog]
-class = ckanext.harvest.log.DBLogHandler
-args = ()
-level = DEBUG
-formatter = dblog
-
-[formatter_dblog]
-format = %(message)s
-
-[formatter_generic]
-format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
--- a/test-core-nose.ini
+++ b/test-core-nose.ini
--- a/test.ini
+++ b/test.ini
@ -1,5 +1,5 @@
 [DEFAULT]
-debug = true
+debug = false
 # Uncomment and replace with the address which should receive any error reports
 #email_to = you@yourdomain.com
 smtp_server = localhost
@ -12,10 +12,12 @@ port = 5000


 [app:main]
-use = config:../ckan/test.ini
+use = config:../ckan/test-core.ini
 # Here we hard-code the database and a flag to make default tests
 # run fast.
-ckan.plugins = harvest ckan_harvester test_harvester
+ckan.plugins = harvest ckan_harvester test_harvester test_harvester2 test_action_harvester
+ckan.harvest.mq.type = redis
+ckan.legacy_templates = false
 # NB: other test configuration should go in test-core.ini, which is
 #     what the postgres tests use.

@ -25,10 +27,10 @@ ckan.plugins = harvest ckan_harvester test_harvester
 keys = root, ckan, sqlalchemy

 [handlers]
-keys = console
+keys = console, dblog

 [formatters]
-keys = generic
+keys = generic, dblog

 [logger_root]
 level = WARN
@ -39,6 +41,11 @@ qualname = ckan
 handlers = 
 level = INFO

+[logger_ckan_harvester]
+qualname = ckanext.harvest
+handlers = dblog
+level = DEBUG
+
 [logger_sqlalchemy]
 handlers =
 qualname = sqlalchemy.engine
@ -50,5 +57,14 @@ args = (sys.stdout,)
 level = NOTSET
 formatter = generic

+[handler_dblog]
+class = ckanext.harvest.log.DBLogHandler
+args = ()
+level = DEBUG
+formatter = dblog
+
+[formatter_dblog]
+format = %(message)s
+
 [formatter_generic]
 format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s