CatalogueRevamping
/
spatial-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Drop support for Python 2

This commit is contained in:
amercader 2023-10-31 15:50:02 +01:00
parent b4e9d97020
commit dee8ffed97
16 changed files with 54 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/test.yml vendored
View File

@ -21,17 +21,11 @@ jobs:
matrix:
include:
- ckan-version: "2.10"
solr-image: "2.10-spatial"
requirements-file: 'requirements.txt'
solr-image: "2.10-solr9-spatial"
harvester-version: 'master'
- ckan-version: 2.9
solr-image: 2.9-solr8-spatial
requirements-file: 'requirements.txt'
solr-image: 2.9-solr9-spatial
harvester-version: 'master'
- ckan-version: 2.9-py2
solr-image: 2.9-py2-solr8-spatial
requirements-file: 'requirements-py2.txt'
harvester-version: 'v1.4.2'
fail-fast: false
name: CKAN ${{ matrix.ckan-version }}, Solr ${{ matrix.solr-image }}
@ -98,16 +92,10 @@ jobs:
pip install cython==0.29.36
pip install --no-use-pep517 pyproj==2.6.1
- name: Patch to test pyproj
if: ${{ matrix.ckan-version == '2.9-py2' }}
run: |
pip install cython==0.28.4
pip install --no-use-pep517 pyproj==2.2.2
- name: Install dependencies from ${{ matrix.requirements-file }}
- name: Install dependencies from requirements.txt
if: steps.cache.outputs.cache-hit != 'true'
run: |
pip install -r ${{ matrix.requirements-file }}
pip install -r requirements.txt
- name: Install harvester
if: steps.cache.outputs.cache-hit != 'true'

2
bin/ckan_pycsw.py
View File

@ -4,7 +4,7 @@ import datetime
import io
import os
import argparse
from six.moves.configparser import SafeConfigParser
from configparser import SafeConfigParser
import requests
from lxml import etree

3
ckanext/spatial/harvested_metadata.py
View File

@ -1,5 +1,4 @@
from lxml import etree
import six
import logging
log = logging.getLogger(__name__)
@ -38,7 +37,7 @@ class MappedXmlDocument(MappedXmlObject):
def get_xml_tree(self):
if self.xml_tree is None:
parser = etree.XMLParser(remove_blank_text=True)
xml_str = six.ensure_str(self.xml_str)
xml_str = str(self.xml_str)
self.xml_tree = etree.fromstring(xml_str, parser=parser)
return self.xml_tree

25
ckanext/spatial/harvesters/base.py
View File

@ -1,6 +1,5 @@
import six
from six.moves.urllib.parse import urlparse
from six.moves.urllib.request import urlopen
from urllib.parse import urlparse
from urllib.request import urlopen
import re
import cgitb
@ -33,7 +32,7 @@ from ckanext.harvest.model import HarvestObject
from ckanext.spatial.validation import Validators, all_validators
from ckanext.spatial.harvested_metadata import ISODocument
from ckanext.spatial.interfaces import ISpatialHarvester
from ckantoolkit import config, unicode_safe
from ckantoolkit import config
log = logging.getLogger(__name__)
@ -300,7 +299,7 @@ class SpatialHarvester(HarvesterBase):
if package is None or package.title != iso_values['title']:
name = self._gen_new_name(iso_values['title'])
if not name:
name = self._gen_new_name(six.text_type(iso_values['guid']))
name = self._gen_new_name(str(iso_values['guid']))
if not name:
raise Exception('Could not generate a unique name from the title or the GUID. Please choose a more unique title.')
package_dict['name'] = name
@ -414,7 +413,7 @@ class SpatialHarvester(HarvesterBase):
ymin = float(bbox['south'])
ymax = float(bbox['north'])
except ValueError as e:
self._save_object_error('Error parsing bounding box value: {0}'.format(six.text_type(e)),
self._save_object_error('Error parsing bounding box value: {0}'.format(str(e)),
harvest_object, 'Import')
else:
# Construct a GeoJSON extent so ckanext-spatial can register the extent geometry
@ -472,7 +471,7 @@ class SpatialHarvester(HarvesterBase):
log.debug('Processing extra %s', key)
if not key in extras or override_extras:
# Look for replacement strings
if isinstance(value,six.string_types):
if isinstance(value,str):
value = value.format(harvest_source_id=harvest_object.job.source.id,
harvest_source_url=harvest_object.job.source.url.strip('/'),
harvest_source_title=harvest_object.job.source.title,
@ -576,7 +575,7 @@ class SpatialHarvester(HarvesterBase):
iso_parser = ISODocument(harvest_object.content)
iso_values = iso_parser.read_values()
except Exception as e:
self._save_object_error('Error parsing ISO document for object {0}: {1}'.format(harvest_object.id, six.text_type(e)),
self._save_object_error('Error parsing ISO document for object {0}: {1}'.format(harvest_object.id, str(e)),
harvest_object, 'Import')
return False
@ -659,7 +658,7 @@ class SpatialHarvester(HarvesterBase):
# We need to explicitly provide a package ID, otherwise ckanext-spatial
# won't be be able to link the extent to the package.
package_dict['id'] = six.text_type(uuid.uuid4())
package_dict['id'] = str(uuid.uuid4())
package_schema['id'] = [unicode_safe]
# Save reference to the package on the object
@ -675,7 +674,7 @@ class SpatialHarvester(HarvesterBase):
package_id = p.toolkit.get_action('package_create')(context, package_dict)
log.info('Created new package %s with guid %s', package_id, harvest_object.guid)
except p.toolkit.ValidationError as e:
self._save_object_error('Validation Error: %s' % six.text_type(e.error_summary), harvest_object, 'Import')
self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import')
return False
elif status == 'change':
@ -721,7 +720,7 @@ class SpatialHarvester(HarvesterBase):
package_id = p.toolkit.get_action('package_update')(context, package_dict)
log.info('Updated package %s with guid %s', package_id, harvest_object.guid)
except p.toolkit.ValidationError as e:
self._save_object_error('Validation Error: %s' % six.text_type(e.error_summary), harvest_object, 'Import')
self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import')
return False
model.Session.commit()
@ -738,7 +737,7 @@ class SpatialHarvester(HarvesterBase):
s = wms.WebMapService(url)
return isinstance(s.contents, dict) and s.contents != {}
except Exception as e:
log.error('WMS check for %s failed with exception: %s' % (url, six.text_type(e)))
log.error('WMS check for %s failed with exception: %s' % (url, str(e)))
return False
def _get_object_extra(self, harvest_object, key):
@ -881,7 +880,7 @@ class SpatialHarvester(HarvesterBase):
try:
xml = etree.fromstring(document_string)
except etree.XMLSyntaxError as e:
self._save_object_error('Could not parse XML file: {0}'.format(six.text_type(e)), harvest_object, 'Import')
self._save_object_error('Could not parse XML file: {0}'.format(str(e)), harvest_object, 'Import')
return False, None, []
valid, profile, errors = validator.is_valid(xml)

5
ckanext/spatial/harvesters/csw.py
View File

@ -1,6 +1,5 @@
import re
import six
from six.moves.urllib.parse import urlparse, urlunparse, urlencode
from urllib.parse import urlparse, urlunparse, urlencode
import logging
@ -105,7 +104,7 @@ class CSWHarvester(SpatialHarvester, SingletonPlugin):
except Exception as e:
log.error('Exception: %s' % text_traceback())
self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % six.text_type(e), harvest_job)
self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % str(e), harvest_job)
return None
new = guids_in_harvest - guids_in_db

26
ckanext/spatial/harvesters/gemini.py
View File

@ -8,9 +8,8 @@ but can be easily adapted for other INSPIRE/ISO19139 XML metadata
- GeminiWafHarvester - An index page with links to GEMINI resources
'''
import six
import os
from six.moves.urllib.parse import urlparse
from urllib.parse import urlparse
from datetime import datetime
from numbers import Number
import uuid
@ -24,6 +23,7 @@ from ckan import model
from ckan.model import Session, Package
from ckan.lib.munge import munge_title_to_name
from ckan.plugins.core import SingletonPlugin, implements
from ckan.lib.navl.validators import unicode_safe
from ckan.lib.helpers import json
from ckan import logic
@ -73,10 +73,10 @@ class GeminiHarvester(SpatialHarvester):
return True
except Exception as e:
log.error('Exception during import: %s' % text_traceback())
if not six.text_type(e).strip():
if not str(e).strip():
self._save_object_error('Error importing Gemini document.', harvest_object, 'Import')
else:
self._save_object_error('Error importing Gemini document: %s' % six.text_type(e), harvest_object, 'Import')
self._save_object_error('Error importing Gemini document: %s' % str(e), harvest_object, 'Import')
raise
if debug_exception_mode:
raise
@ -275,7 +275,7 @@ class GeminiHarvester(SpatialHarvester):
if package is None or package.title != gemini_values['title']:
name = self.gen_new_name(gemini_values['title'])
if not name:
name = self.gen_new_name(six.text_type(gemini_guid))
name = self.gen_new_name(str(gemini_guid))
if not name:
raise Exception('Could not generate a unique name from the title or the GUID. Please choose a more unique title.')
package_dict['name'] = name
@ -320,7 +320,7 @@ class GeminiHarvester(SpatialHarvester):
extras_as_dict = []
for key,value in extras.items():
if isinstance(value, six.string_types + (Number,)):
if isinstance(value, str + (Number,)):
extras_as_dict.append({'key':key,'value':value})
else:
extras_as_dict.append({'key':key,'value':json.dumps(value)})
@ -413,8 +413,8 @@ class GeminiHarvester(SpatialHarvester):
else:
counter = 1
while counter < 101:
if name+six.text_type(counter) not in taken:
return name+six.text_type(counter)
if name+str(counter) not in taken:
return name+str(counter)
counter = counter + 1
return None
@ -454,7 +454,7 @@ class GeminiHarvester(SpatialHarvester):
# The default package schema does not like Upper case tags
tag_schema = logic.schema.default_tags_schema()
tag_schema['name'] = [not_empty,six.text_type]
tag_schema['name'] = [not_empty, unicode_safe]
package_schema['tags'] = tag_schema
# TODO: user
@ -467,8 +467,8 @@ class GeminiHarvester(SpatialHarvester):
if not package:
# We need to explicitly provide a package ID, otherwise ckanext-spatial
# won't be be able to link the extent to the package.
package_dict['id'] = six.text_type(uuid.uuid4())
package_schema['id'] = [six.text_type]
package_dict['id'] = str(uuid.uuid4())
package_schema['id'] = [unicode_safe]
action_function = get_action('package_create')
else:
@ -478,7 +478,7 @@ class GeminiHarvester(SpatialHarvester):
try:
package_dict = action_function(context, package_dict)
except ValidationError as e:
raise Exception('Validation Error: %s' % six.text_type(e.error_summary))
raise Exception('Validation Error: %s' % str(e.error_summary))
if debug_exception_mode:
raise
@ -572,7 +572,7 @@ class GeminiCswHarvester(GeminiHarvester, SingletonPlugin):
except Exception as e:
log.error('Exception: %s' % text_traceback())
self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % six.text_type(e), harvest_job)
self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % str(e), harvest_job)
return None
if len(ids) == 0:

11
ckanext/spatial/harvesters/waf.py
View File

@ -1,7 +1,6 @@
from __future__ import print_function
import six
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
import logging
import hashlib
@ -98,7 +97,7 @@ class WAFHarvester(SpatialHarvester, SingletonPlugin):
url_to_modified_harvest = {} ## mapping of url to last_modified in harvest
try:
for url, modified_date in _extract_waf(six.text_type(content),source_url,scraper):
for url, modified_date in _extract_waf(str(content),source_url,scraper):
url_to_modified_harvest[url] = modified_date
except Exception as e:
msg = 'Error extracting URLs from %s, error was %s' % (source_url, e)
@ -316,16 +315,16 @@ def _extract_waf(content, base_url, scraper, results = None, depth=0):
response = requests.get(new_url)
content = response.content
except Exception as e:
print(six.text_type(e))
print(str(e))
continue
_extract_waf(six.text_type(content), new_url, scraper, results, new_depth)
_extract_waf(str(content), new_url, scraper, results, new_depth)
continue
if not url.endswith('.xml'):
continue
date = record.date
if date:
try:
date = six.text_type(dateutil.parser.parse(date))
date = str(dateutil.parser.parse(date))
except Exception as e:
raise
date = None

3
ckanext/spatial/lib/__init__.py
View File

@ -1,5 +1,4 @@
import logging
import six
import ckantoolkit as tk
config = tk.config
@ -47,7 +46,7 @@ def normalize_bbox(bbox_values):
If there are any problems parsing the input it returns None.
"""
if isinstance(bbox_values, six.string_types):
if isinstance(bbox_values, str):
bbox_values = bbox_values.split(",")
if len(bbox_values) != 4:

3
ckanext/spatial/lib/csw_client.py
View File

@ -2,7 +2,6 @@
Some very thin wrapper classes around those in OWSLib
for convenience.
"""
import six
import logging
from owslib.etree import etree
@ -33,7 +32,7 @@ class OwsService(object):
pass
elif callable(val):
pass
elif isinstance(val, six.string_types):
elif isinstance(val, str):
md[attr] = val
elif isinstance(val, int):
md[attr] = val

6
ckanext/spatial/lib/report.py
View File

@ -3,7 +3,7 @@ Library for creating reports that can be displayed easily in an HTML table
and then saved as a CSV.
'''
from six import text_type, StringIO
from io import StringIO
import datetime
import csv
@ -52,9 +52,9 @@ class ReportTable(object):
if isinstance(cell, datetime.datetime):
cell = cell.strftime('%Y-%m-%d %H:%M')
elif isinstance(cell, int):
cell = text_type(cell)
cell = str(cell)
elif isinstance(cell, (list, tuple)):
cell = text_type(cell)
cell = str(cell)
elif cell is None:
cell = ''
else:

5
ckanext/spatial/plugin/__init__.py
View File

@ -2,7 +2,6 @@ import os
import mimetypes
from logging import getLogger
import six
import geojson
import shapely.geometry
@ -104,10 +103,10 @@ class SpatialMetadata(p.SingletonPlugin):
try:
log.debug("Received geometry: {}".format(geometry))
geometry = geojson.loads(six.text_type(geometry))
geometry = geojson.loads(str(geometry))
except ValueError as e:
error_dict = {
"spatial": ["Error decoding JSON object: {}".format(six.text_type(e))]}
"spatial": ["Error decoding JSON object: {}".format(str(e))]}
raise tk.ValidationError(error_dict)
if not hasattr(geometry, "is_valid") or not geometry.is_valid:

7
ckanext/spatial/util.py
View File

@ -3,8 +3,7 @@
from __future__ import print_function
import os
import sys
import six
from io import StringIO
from pkg_resources import resource_stream
import logging
@ -32,7 +31,7 @@ log = logging.getLogger(__name__)
def report(pkg=None):
if pkg:
package_ref = six.text_type(pkg)
package_ref = str(pkg)
pkg = model.Package.get(package_ref)
if not pkg:
print('Package ref "%s" not recognised' % package_ref)
@ -153,7 +152,7 @@ def transform_to_html(content, xslt_package=None, xslt_path=None):
style_xml = etree.parse(style)
transformer = etree.XSLT(style_xml)
xml = etree.parse(six.StringIO(content and six.text_type(content)))
xml = etree.parse(StringIO(content and str(content)))
html = transformer(xml)
result = etree.tostring(html, pretty_print=True)

3
doc/spatial-search.rst
View File

@ -172,6 +172,9 @@ details about the available options (again, you don't need to modify Solr if you
"{{!field f=spatial_geom}}Intersects(ENVELOPE({minx}, {maxx}, {maxy}, {miny}))"
.. note:: The old ``postgis`` search backend is no longer supported. You should migrate to one of the other backends instead.
Spatial Search Widget
---------------------

14
requirements-py2.txt
View File

@ -1,14 +0,0 @@
ckantoolkit
cython==0.29.36
Shapely>=1.2.13,<2.0.0
pyproj==2.2.2
OWSLib==0.18.0
lxml>=2.3
argparse
pyparsing>=2.1.10
requests>=1.1.0
six
geojson==2.5.0
# Harvest extension install 1.3 which try to install
# setuptools>=61.2 which is not compatible with python 2.7
pika==1.1.0

1
requirements.txt
View File

@ -3,7 +3,6 @@ lxml>=2.3
argparse
pyparsing>=2.1.10
requests>=1.1.0
six
cython==0.29.36; python_version < '3.9'
pyproj==2.6.1; python_version < '3.9'
pyproj @ git+https://github.com/pyproj4/pyproj.git@main; python_version >= '3.9'

5
setup.py
View File

@ -28,10 +28,11 @@ https://docs.ckan.org/projects/ckanext-spatial/en/latest/
"Development Status :: 5 - Production/Stable",
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
"Programming Language :: Python",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
],
keywords="",
author="Open Knowledge Foundation",