diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index abc2340..5f3fb00 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,17 +21,11 @@ jobs: matrix: include: - ckan-version: "2.10" - solr-image: "2.10-spatial" - requirements-file: 'requirements.txt' + solr-image: "2.10-solr9-spatial" harvester-version: 'master' - ckan-version: 2.9 - solr-image: 2.9-solr8-spatial - requirements-file: 'requirements.txt' + solr-image: 2.9-solr9-spatial harvester-version: 'master' - - ckan-version: 2.9-py2 - solr-image: 2.9-py2-solr8-spatial - requirements-file: 'requirements-py2.txt' - harvester-version: 'v1.4.2' fail-fast: false name: CKAN ${{ matrix.ckan-version }}, Solr ${{ matrix.solr-image }} @@ -98,16 +92,10 @@ jobs: pip install cython==0.29.36 pip install --no-use-pep517 pyproj==2.6.1 - - name: Patch to test pyproj - if: ${{ matrix.ckan-version == '2.9-py2' }} - run: | - pip install cython==0.28.4 - pip install --no-use-pep517 pyproj==2.2.2 - - - name: Install dependencies from ${{ matrix.requirements-file }} + - name: Install dependencies from requirements.txt if: steps.cache.outputs.cache-hit != 'true' run: | - pip install -r ${{ matrix.requirements-file }} + pip install -r requirements.txt - name: Install harvester if: steps.cache.outputs.cache-hit != 'true' diff --git a/bin/ckan_pycsw.py b/bin/ckan_pycsw.py index 47e171b..05479e2 100644 --- a/bin/ckan_pycsw.py +++ b/bin/ckan_pycsw.py @@ -4,7 +4,7 @@ import datetime import io import os import argparse -from six.moves.configparser import SafeConfigParser +from configparser import SafeConfigParser import requests from lxml import etree diff --git a/ckanext/spatial/harvested_metadata.py b/ckanext/spatial/harvested_metadata.py index a722518..4520ca2 100644 --- a/ckanext/spatial/harvested_metadata.py +++ b/ckanext/spatial/harvested_metadata.py @@ -1,5 +1,4 @@ from lxml import etree -import six import logging log = logging.getLogger(__name__) @@ -38,7 +37,7 @@ class MappedXmlDocument(MappedXmlObject): def get_xml_tree(self): if self.xml_tree is None: parser = etree.XMLParser(remove_blank_text=True) - xml_str = six.ensure_str(self.xml_str) + xml_str = str(self.xml_str) self.xml_tree = etree.fromstring(xml_str, parser=parser) return self.xml_tree diff --git a/ckanext/spatial/harvesters/base.py b/ckanext/spatial/harvesters/base.py index 9613a7e..ecf5e67 100644 --- a/ckanext/spatial/harvesters/base.py +++ b/ckanext/spatial/harvesters/base.py @@ -1,6 +1,5 @@ -import six -from six.moves.urllib.parse import urlparse -from six.moves.urllib.request import urlopen +from urllib.parse import urlparse +from urllib.request import urlopen import re import cgitb @@ -33,7 +32,7 @@ from ckanext.harvest.model import HarvestObject from ckanext.spatial.validation import Validators, all_validators from ckanext.spatial.harvested_metadata import ISODocument from ckanext.spatial.interfaces import ISpatialHarvester -from ckantoolkit import config, unicode_safe +from ckantoolkit import config log = logging.getLogger(__name__) @@ -300,7 +299,7 @@ class SpatialHarvester(HarvesterBase): if package is None or package.title != iso_values['title']: name = self._gen_new_name(iso_values['title']) if not name: - name = self._gen_new_name(six.text_type(iso_values['guid'])) + name = self._gen_new_name(str(iso_values['guid'])) if not name: raise Exception('Could not generate a unique name from the title or the GUID. Please choose a more unique title.') package_dict['name'] = name @@ -414,7 +413,7 @@ class SpatialHarvester(HarvesterBase): ymin = float(bbox['south']) ymax = float(bbox['north']) except ValueError as e: - self._save_object_error('Error parsing bounding box value: {0}'.format(six.text_type(e)), + self._save_object_error('Error parsing bounding box value: {0}'.format(str(e)), harvest_object, 'Import') else: # Construct a GeoJSON extent so ckanext-spatial can register the extent geometry @@ -472,7 +471,7 @@ class SpatialHarvester(HarvesterBase): log.debug('Processing extra %s', key) if not key in extras or override_extras: # Look for replacement strings - if isinstance(value,six.string_types): + if isinstance(value,str): value = value.format(harvest_source_id=harvest_object.job.source.id, harvest_source_url=harvest_object.job.source.url.strip('/'), harvest_source_title=harvest_object.job.source.title, @@ -576,7 +575,7 @@ class SpatialHarvester(HarvesterBase): iso_parser = ISODocument(harvest_object.content) iso_values = iso_parser.read_values() except Exception as e: - self._save_object_error('Error parsing ISO document for object {0}: {1}'.format(harvest_object.id, six.text_type(e)), + self._save_object_error('Error parsing ISO document for object {0}: {1}'.format(harvest_object.id, str(e)), harvest_object, 'Import') return False @@ -659,7 +658,7 @@ class SpatialHarvester(HarvesterBase): # We need to explicitly provide a package ID, otherwise ckanext-spatial # won't be be able to link the extent to the package. - package_dict['id'] = six.text_type(uuid.uuid4()) + package_dict['id'] = str(uuid.uuid4()) package_schema['id'] = [unicode_safe] # Save reference to the package on the object @@ -675,7 +674,7 @@ class SpatialHarvester(HarvesterBase): package_id = p.toolkit.get_action('package_create')(context, package_dict) log.info('Created new package %s with guid %s', package_id, harvest_object.guid) except p.toolkit.ValidationError as e: - self._save_object_error('Validation Error: %s' % six.text_type(e.error_summary), harvest_object, 'Import') + self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import') return False elif status == 'change': @@ -721,7 +720,7 @@ class SpatialHarvester(HarvesterBase): package_id = p.toolkit.get_action('package_update')(context, package_dict) log.info('Updated package %s with guid %s', package_id, harvest_object.guid) except p.toolkit.ValidationError as e: - self._save_object_error('Validation Error: %s' % six.text_type(e.error_summary), harvest_object, 'Import') + self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import') return False model.Session.commit() @@ -738,7 +737,7 @@ class SpatialHarvester(HarvesterBase): s = wms.WebMapService(url) return isinstance(s.contents, dict) and s.contents != {} except Exception as e: - log.error('WMS check for %s failed with exception: %s' % (url, six.text_type(e))) + log.error('WMS check for %s failed with exception: %s' % (url, str(e))) return False def _get_object_extra(self, harvest_object, key): @@ -881,7 +880,7 @@ class SpatialHarvester(HarvesterBase): try: xml = etree.fromstring(document_string) except etree.XMLSyntaxError as e: - self._save_object_error('Could not parse XML file: {0}'.format(six.text_type(e)), harvest_object, 'Import') + self._save_object_error('Could not parse XML file: {0}'.format(str(e)), harvest_object, 'Import') return False, None, [] valid, profile, errors = validator.is_valid(xml) diff --git a/ckanext/spatial/harvesters/csw.py b/ckanext/spatial/harvesters/csw.py index c27824b..e2c0448 100644 --- a/ckanext/spatial/harvesters/csw.py +++ b/ckanext/spatial/harvesters/csw.py @@ -1,6 +1,5 @@ import re -import six -from six.moves.urllib.parse import urlparse, urlunparse, urlencode +from urllib.parse import urlparse, urlunparse, urlencode import logging @@ -105,7 +104,7 @@ class CSWHarvester(SpatialHarvester, SingletonPlugin): except Exception as e: log.error('Exception: %s' % text_traceback()) - self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % six.text_type(e), harvest_job) + self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % str(e), harvest_job) return None new = guids_in_harvest - guids_in_db diff --git a/ckanext/spatial/harvesters/gemini.py b/ckanext/spatial/harvesters/gemini.py index 07fa0b9..af00112 100644 --- a/ckanext/spatial/harvesters/gemini.py +++ b/ckanext/spatial/harvesters/gemini.py @@ -8,9 +8,8 @@ but can be easily adapted for other INSPIRE/ISO19139 XML metadata - GeminiWafHarvester - An index page with links to GEMINI resources ''' -import six import os -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from datetime import datetime from numbers import Number import uuid @@ -24,6 +23,7 @@ from ckan import model from ckan.model import Session, Package from ckan.lib.munge import munge_title_to_name from ckan.plugins.core import SingletonPlugin, implements +from ckan.lib.navl.validators import unicode_safe from ckan.lib.helpers import json from ckan import logic @@ -73,10 +73,10 @@ class GeminiHarvester(SpatialHarvester): return True except Exception as e: log.error('Exception during import: %s' % text_traceback()) - if not six.text_type(e).strip(): + if not str(e).strip(): self._save_object_error('Error importing Gemini document.', harvest_object, 'Import') else: - self._save_object_error('Error importing Gemini document: %s' % six.text_type(e), harvest_object, 'Import') + self._save_object_error('Error importing Gemini document: %s' % str(e), harvest_object, 'Import') raise if debug_exception_mode: raise @@ -275,7 +275,7 @@ class GeminiHarvester(SpatialHarvester): if package is None or package.title != gemini_values['title']: name = self.gen_new_name(gemini_values['title']) if not name: - name = self.gen_new_name(six.text_type(gemini_guid)) + name = self.gen_new_name(str(gemini_guid)) if not name: raise Exception('Could not generate a unique name from the title or the GUID. Please choose a more unique title.') package_dict['name'] = name @@ -320,7 +320,7 @@ class GeminiHarvester(SpatialHarvester): extras_as_dict = [] for key,value in extras.items(): - if isinstance(value, six.string_types + (Number,)): + if isinstance(value, str + (Number,)): extras_as_dict.append({'key':key,'value':value}) else: extras_as_dict.append({'key':key,'value':json.dumps(value)}) @@ -413,8 +413,8 @@ class GeminiHarvester(SpatialHarvester): else: counter = 1 while counter < 101: - if name+six.text_type(counter) not in taken: - return name+six.text_type(counter) + if name+str(counter) not in taken: + return name+str(counter) counter = counter + 1 return None @@ -454,7 +454,7 @@ class GeminiHarvester(SpatialHarvester): # The default package schema does not like Upper case tags tag_schema = logic.schema.default_tags_schema() - tag_schema['name'] = [not_empty,six.text_type] + tag_schema['name'] = [not_empty, unicode_safe] package_schema['tags'] = tag_schema # TODO: user @@ -467,8 +467,8 @@ class GeminiHarvester(SpatialHarvester): if not package: # We need to explicitly provide a package ID, otherwise ckanext-spatial # won't be be able to link the extent to the package. - package_dict['id'] = six.text_type(uuid.uuid4()) - package_schema['id'] = [six.text_type] + package_dict['id'] = str(uuid.uuid4()) + package_schema['id'] = [unicode_safe] action_function = get_action('package_create') else: @@ -478,7 +478,7 @@ class GeminiHarvester(SpatialHarvester): try: package_dict = action_function(context, package_dict) except ValidationError as e: - raise Exception('Validation Error: %s' % six.text_type(e.error_summary)) + raise Exception('Validation Error: %s' % str(e.error_summary)) if debug_exception_mode: raise @@ -572,7 +572,7 @@ class GeminiCswHarvester(GeminiHarvester, SingletonPlugin): except Exception as e: log.error('Exception: %s' % text_traceback()) - self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % six.text_type(e), harvest_job) + self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % str(e), harvest_job) return None if len(ids) == 0: diff --git a/ckanext/spatial/harvesters/waf.py b/ckanext/spatial/harvesters/waf.py index 1dd96de..5338d87 100644 --- a/ckanext/spatial/harvesters/waf.py +++ b/ckanext/spatial/harvesters/waf.py @@ -1,7 +1,6 @@ from __future__ import print_function -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin import logging import hashlib @@ -98,7 +97,7 @@ class WAFHarvester(SpatialHarvester, SingletonPlugin): url_to_modified_harvest = {} ## mapping of url to last_modified in harvest try: - for url, modified_date in _extract_waf(six.text_type(content),source_url,scraper): + for url, modified_date in _extract_waf(str(content),source_url,scraper): url_to_modified_harvest[url] = modified_date except Exception as e: msg = 'Error extracting URLs from %s, error was %s' % (source_url, e) @@ -316,16 +315,16 @@ def _extract_waf(content, base_url, scraper, results = None, depth=0): response = requests.get(new_url) content = response.content except Exception as e: - print(six.text_type(e)) + print(str(e)) continue - _extract_waf(six.text_type(content), new_url, scraper, results, new_depth) + _extract_waf(str(content), new_url, scraper, results, new_depth) continue if not url.endswith('.xml'): continue date = record.date if date: try: - date = six.text_type(dateutil.parser.parse(date)) + date = str(dateutil.parser.parse(date)) except Exception as e: raise date = None diff --git a/ckanext/spatial/lib/__init__.py b/ckanext/spatial/lib/__init__.py index 2e3fcaa..8792ec6 100644 --- a/ckanext/spatial/lib/__init__.py +++ b/ckanext/spatial/lib/__init__.py @@ -1,5 +1,4 @@ import logging -import six import ckantoolkit as tk config = tk.config @@ -47,7 +46,7 @@ def normalize_bbox(bbox_values): If there are any problems parsing the input it returns None. """ - if isinstance(bbox_values, six.string_types): + if isinstance(bbox_values, str): bbox_values = bbox_values.split(",") if len(bbox_values) != 4: diff --git a/ckanext/spatial/lib/csw_client.py b/ckanext/spatial/lib/csw_client.py index b553344..8a4d325 100644 --- a/ckanext/spatial/lib/csw_client.py +++ b/ckanext/spatial/lib/csw_client.py @@ -2,7 +2,6 @@ Some very thin wrapper classes around those in OWSLib for convenience. """ -import six import logging from owslib.etree import etree @@ -33,7 +32,7 @@ class OwsService(object): pass elif callable(val): pass - elif isinstance(val, six.string_types): + elif isinstance(val, str): md[attr] = val elif isinstance(val, int): md[attr] = val diff --git a/ckanext/spatial/lib/report.py b/ckanext/spatial/lib/report.py index d1165d6..d623544 100644 --- a/ckanext/spatial/lib/report.py +++ b/ckanext/spatial/lib/report.py @@ -3,7 +3,7 @@ Library for creating reports that can be displayed easily in an HTML table and then saved as a CSV. ''' -from six import text_type, StringIO +from io import StringIO import datetime import csv @@ -52,9 +52,9 @@ class ReportTable(object): if isinstance(cell, datetime.datetime): cell = cell.strftime('%Y-%m-%d %H:%M') elif isinstance(cell, int): - cell = text_type(cell) + cell = str(cell) elif isinstance(cell, (list, tuple)): - cell = text_type(cell) + cell = str(cell) elif cell is None: cell = '' else: diff --git a/ckanext/spatial/plugin/__init__.py b/ckanext/spatial/plugin/__init__.py index 158c009..f8c8968 100644 --- a/ckanext/spatial/plugin/__init__.py +++ b/ckanext/spatial/plugin/__init__.py @@ -2,7 +2,6 @@ import os import mimetypes from logging import getLogger -import six import geojson import shapely.geometry @@ -104,10 +103,10 @@ class SpatialMetadata(p.SingletonPlugin): try: log.debug("Received geometry: {}".format(geometry)) - geometry = geojson.loads(six.text_type(geometry)) + geometry = geojson.loads(str(geometry)) except ValueError as e: error_dict = { - "spatial": ["Error decoding JSON object: {}".format(six.text_type(e))]} + "spatial": ["Error decoding JSON object: {}".format(str(e))]} raise tk.ValidationError(error_dict) if not hasattr(geometry, "is_valid") or not geometry.is_valid: diff --git a/ckanext/spatial/util.py b/ckanext/spatial/util.py index 50f061a..fc822d2 100644 --- a/ckanext/spatial/util.py +++ b/ckanext/spatial/util.py @@ -3,8 +3,7 @@ from __future__ import print_function import os import sys - -import six +from io import StringIO from pkg_resources import resource_stream import logging @@ -32,7 +31,7 @@ log = logging.getLogger(__name__) def report(pkg=None): if pkg: - package_ref = six.text_type(pkg) + package_ref = str(pkg) pkg = model.Package.get(package_ref) if not pkg: print('Package ref "%s" not recognised' % package_ref) @@ -153,7 +152,7 @@ def transform_to_html(content, xslt_package=None, xslt_path=None): style_xml = etree.parse(style) transformer = etree.XSLT(style_xml) - xml = etree.parse(six.StringIO(content and six.text_type(content))) + xml = etree.parse(StringIO(content and str(content))) html = transformer(xml) result = etree.tostring(html, pretty_print=True) diff --git a/doc/spatial-search.rst b/doc/spatial-search.rst index c403573..a64fec7 100644 --- a/doc/spatial-search.rst +++ b/doc/spatial-search.rst @@ -172,6 +172,9 @@ details about the available options (again, you don't need to modify Solr if you "{{!field f=spatial_geom}}Intersects(ENVELOPE({minx}, {maxx}, {maxy}, {miny}))" +.. note:: The old ``postgis`` search backend is no longer supported. You should migrate to one of the other backends instead. + + Spatial Search Widget --------------------- diff --git a/requirements-py2.txt b/requirements-py2.txt deleted file mode 100644 index 8eff582..0000000 --- a/requirements-py2.txt +++ /dev/null @@ -1,14 +0,0 @@ -ckantoolkit -cython==0.29.36 -Shapely>=1.2.13,<2.0.0 -pyproj==2.2.2 -OWSLib==0.18.0 -lxml>=2.3 -argparse -pyparsing>=2.1.10 -requests>=1.1.0 -six -geojson==2.5.0 -# Harvest extension install 1.3 which try to install -# setuptools>=61.2 which is not compatible with python 2.7 -pika==1.1.0 diff --git a/requirements.txt b/requirements.txt index c0c58d1..c025b11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ lxml>=2.3 argparse pyparsing>=2.1.10 requests>=1.1.0 -six cython==0.29.36; python_version < '3.9' pyproj==2.6.1; python_version < '3.9' pyproj @ git+https://github.com/pyproj4/pyproj.git@main; python_version >= '3.9' diff --git a/setup.py b/setup.py index 5bef419..ae5cd1e 100644 --- a/setup.py +++ b/setup.py @@ -28,10 +28,11 @@ https://docs.ckan.org/projects/ckanext-spatial/en/latest/ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", "Programming Language :: Python", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], keywords="", author="Open Knowledge Foundation",