Py3 fixes, mostly bytes/str ones

ckanext/harvest/harvesters/ckanharvester.py

@@ -1,10 +1,10 @@
 from __future__ import absolute_import
+import six
 import requests
 from requests.exceptions import HTTPError, RequestException
 
 import datetime
 from urllib3.contrib import pyopenssl
-import urllib
 
 from six.moves.urllib.parse import urlencode
 from ckan import model
@@ -123,7 +123,7 @@ class CKANHarvester(HarvesterBase):
                                      ' names/ids')
                 if config_obj['default_groups'] and \
                         not isinstance(config_obj['default_groups'][0],
-                                       basestring):
+                                       six.string_types):
                     raise ValueError('default_groups must be a list of group '
                                      'names/ids (i.e. strings)')
 
@@ -523,7 +523,7 @@ class CKANHarvester(HarvesterBase):
                     if existing_extra:
                         package_dict['extras'].remove(existing_extra)
                     # Look for replacement strings
-                    if isinstance(value, basestring):
+                    if isinstance(value, six.string_types):
                         value = value.format(
                             harvest_source_id=harvest_object.job.source.id,
                             harvest_source_url=harvest_object.job.source.url.strip('/'),

ckanext/harvest/logic/validators.py

@@ -13,6 +13,7 @@ from ckanext.harvest.utils import (
 from ckanext.harvest.model import HarvestSource, UPDATE_FREQUENCIES, HarvestJob
 from ckanext.harvest.interfaces import IHarvester
 
+import six
 from six.moves.urllib.parse import (
     urlparse, urlunparse
 )
@@ -227,7 +228,7 @@ def harvest_source_convert_from_config(key, data, errors, context):
 
 
 def harvest_source_active_validator(value, context):
-    if isinstance(value, basestring):
+    if isinstance(value, six.string_types):
         if value.lower() == 'true':
             return True
         else:
@@ -253,6 +254,6 @@ def harvest_object_extras_validator(value, context):
     if not isinstance(value, dict):
         raise Invalid('extras must be a dict')
     for v in value.values():
-        if not isinstance(v, basestring):
+        if not isinstance(v, six.string_types):
             raise Invalid('extras must be a dict of strings')
     return value

ckanext/harvest/tests/harvesters/mock_ckan.py

@@ -5,6 +5,7 @@ import re
 import copy
 import urllib
 import six
+from six.moves.urllib.parse import unquote_plus
 
 from threading import Thread
 
@@ -171,7 +172,10 @@ class MockCkanHandler(SimpleHTTPRequestHandler):
 
     def get_url_params(self):
         params_str = self.path.split('?')[-1]
-        params_unicode = urllib.unquote_plus(params_str).decode('utf8')
+        if six.PY2:
+            params_unicode = unquote_plus(params_str).decode('utf8')
+        else:
+            params_unicode = unquote_plus(params_str)
         params = params_unicode.split('&')
         return dict([param.split('=') for param in params])
 
@@ -187,7 +191,7 @@ class MockCkanHandler(SimpleHTTPRequestHandler):
         self.send_response(status)
         self.send_header('Content-Type', content_type)
         self.end_headers()
-        self.wfile.write(content)
+        self.wfile.write(content.encode('utf-8'))
         self.wfile.close()
 
 

ckanext/harvest/tests/test_action.py

@@ -110,7 +110,7 @@ class HarvestSourceActionBase():
         with pytest.raises(ValidationError) as e:
             helpers.call_action(self.action, **source_dict)
 
-        assert u'Error parsing the configuration options: No JSON object could be decoded' in e.value.error_dict['config'][0]
+        assert u'Error parsing the configuration options' in e.value.error_dict['config'][0]
 
         source_dict['config'] = json.dumps({'custom_option': 'not_a_list'})
 
@@ -298,7 +298,8 @@ class TestActions():
             context, {})
 
         # verify
-        assert sorted(result) == sorted([{'id': source_1.id}, {'id': source_2.id}])
+        assert sorted(result, key=lambda item: item['id']) == sorted(
+            [{'id': source_1.id}, {'id': source_2.id}], key=lambda item: item['id'])
         source_1 = harvest_model.HarvestSource.get(source_1.id)
         assert source_1
         assert harvest_model.HarvestJob.get(job_1.id) is None
@@ -368,9 +369,9 @@ class TestActions():
         user = ckan_factories.User()
         user['capacity'] = 'admin'
         org = ckan_factories.Organization(users=[user])
-        source_dict = dict(
-            SOURCE_DICT.items() + [('publisher_id', org['id'])]
-        )
+        source_dict = SOURCE_DICT.copy()
+        source_dict['publisher_id'] = org['id']
+
         source = factories.HarvestSource(**source_dict)
 
         data_dict = {

ckanext/harvest/tests/test_queue.py

@@ -1,4 +1,5 @@
 import pytest
+import six
 from mock import patch
 
 from ckanext.harvest.model import HarvestObject, HarvestObjectExtra
@@ -332,7 +333,12 @@ class TestHarvestCorruptRedis(object):
 
             assert mock_log_error.call_count == 1
             args, _ = mock_log_error.call_args_list[0]
-            assert "cannot concatenate 'str' and 'NoneType' objects" in args[1]
+            if six.PY2:
+                assert "cannot concatenate 'str' and 'NoneType' objects" in args[1]
+            else:
+                assert "must be str, not NoneType" in str(args[1])
+
+
 
         finally:
             redis.delete('ckanext-harvest:some-random-key-2')