diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..af2fbf8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +language: python +python: + - "2.7" +env: + - CKANVERSION=master POSTGISVERSION=1 + - CKANVERSION=master POSTGISVERSION=2 + - CKANVERSION=2.2 POSTGISVERSION=1 + - CKANVERSION=2.2 POSTGISVERSION=2 + - CKANVERSION=2.3 POSTGISVERSION=1 + - CKANVERSION=2.3 POSTGISVERSION=2 +install: + - bash bin/travis-build.bash +script: sh bin/travis-run.sh diff --git a/bin/travis-build.bash b/bin/travis-build.bash new file mode 100644 index 0000000..286d283 --- /dev/null +++ b/bin/travis-build.bash @@ -0,0 +1,93 @@ +#!/bin/bash +set -e + +echo "This is travis-build.bash..." + +echo "Installing the packages that CKAN requires..." +sudo apt-get update -qq +sudo apt-get install postgresql-9.1 solr-jetty libcommons-fileupload-java:amd64=1.2.2-1 + +echo "Installing PostGIS..." +if [ $POSTGISVERSION == '1' ] +then + sudo apt-get install postgresql-9.1-postgis=1.5.3-2 +fi +# PostGIS 2.1 already installed on Travis + +echo "Patching lxml..." +wget ftp://xmlsoft.org/libxml2/libxml2-2.9.0.tar.gz +tar zxf libxml2-2.9.0.tar.gz +cd libxml2-2.9.0/ +./configure --quiet --libdir=/usr/lib/x86_64-linux-gnu +make --silent +sudo make --silent install +xmllint --version +cd - + +echo "Installing CKAN and its Python dependencies..." +git clone https://github.com/ckan/ckan +cd ckan +if [ $CKANVERSION == '2.3' ] +then + git checkout release-v2.3 +elif [ $CKANVERSION == '2.2' ] +then + git checkout release-v2.2.3 +fi +python setup.py develop +pip install -r requirements.txt --allow-all-external +pip install -r dev-requirements.txt --allow-all-external +cd - + +echo "Setting up Solr..." +printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty +sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml +sudo service jetty restart + +echo "Creating the PostgreSQL user and database..." +sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" +sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' + +echo "Setting up PostGIS on the database..." +if [ $POSTGISVERSION == '1' ] +then + sudo -u postgres psql -d ckan_test -f /usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql + sudo -u postgres psql -d ckan_test -f /usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql + sudo -u postgres psql -d ckan_test -c 'ALTER TABLE geometry_columns OWNER TO ckan_default;' + sudo -u postgres psql -d ckan_test -c 'ALTER TABLE spatial_ref_sys OWNER TO ckan_default;' +elif [ $POSTGISVERSION == '2' ] +then + sudo -u postgres psql -d ckan_test -c 'CREATE EXTENSION postgis;' + sudo -u postgres psql -d ckan_test -c 'ALTER VIEW geometry_columns OWNER TO ckan_default;' + sudo -u postgres psql -d ckan_test -c 'ALTER TABLE spatial_ref_sys OWNER TO ckan_default;' +fi + +echo "Install other libraries required..." +sudo apt-get install python-dev libxml2-dev libxslt1-dev libgeos-c1 + +echo "Initialising the database..." +cd ckan +paster db init -c test-core.ini +cd - + +echo "Installing ckanext-harvest and its requirements..." +git clone https://github.com/ckan/ckanext-harvest +cd ckanext-harvest +python setup.py develop +pip install -r pip-requirements.txt --allow-all-external + +paster harvester initdb -c ../ckan/test-core.ini +cd - + +echo "Installing ckanext-spatial and its requirements..." +pip install -r pip-requirements.txt --allow-all-external +python setup.py develop + + +echo "Moving test.ini into a subdir..." +mkdir subdir +mv test.ini subdir + +paster spatial initdb -c subdir/test.ini + +echo "travis-build.bash is done." diff --git a/bin/travis-run.sh b/bin/travis-run.sh new file mode 100644 index 0000000..cd13759 --- /dev/null +++ b/bin/travis-run.sh @@ -0,0 +1,3 @@ +#!/bin/sh -e + +nosetests --ckan --nologcapture --with-pylons=subdir/test.ini ckanext/spatial diff --git a/ckanext/spatial/controllers/api.py b/ckanext/spatial/controllers/api.py index 3988517..a25cc59 100644 --- a/ckanext/spatial/controllers/api.py +++ b/ckanext/spatial/controllers/api.py @@ -3,7 +3,6 @@ import logging try: from cStringIO import StringIO except ImportError: from StringIO import StringIO -from geoalchemy import WKTSpatialElement, functions from pylons import response from pkg_resources import resource_stream from lxml import etree diff --git a/ckanext/spatial/geoalchemy_common.py b/ckanext/spatial/geoalchemy_common.py new file mode 100644 index 0000000..308455d --- /dev/null +++ b/ckanext/spatial/geoalchemy_common.py @@ -0,0 +1,84 @@ +''' +Common codebase for GeoAlchemy and GeoAlchemy2 + +It is assumed that the relevant library is already installed, as we check +it against the CKAN version on startup +''' + +from ckan.plugins import toolkit +from ckan.model import meta, Session + +from sqlalchemy import types, Column, Table + +if toolkit.check_ckan_version(min_version='2.3'): + # CKAN >= 2.3, use GeoAlchemy2 + + from geoalchemy2.elements import WKTElement + from geoalchemy2 import Geometry + from sqlalchemy import func + ST_Transform = func.ST_Transform + ST_Equals = func.ST_Equals + + legacy_geoalchemy = False +else: + # CKAN < 2.3, use GeoAlchemy + + from geoalchemy import WKTSpatialElement as WKTElement + from geoalchemy import functions + ST_Transform = functions.transform + ST_Equals = functions.equals + + from geoalchemy import (Geometry, GeometryColumn, GeometryDDL, + GeometryExtensionColumn) + from geoalchemy.postgis import PGComparator + + legacy_geoalchemy = True + + +def postgis_version(): + + result = Session.execute('SELECT postgis_lib_version()') + + return result.scalar() + + +def setup_spatial_table(package_extent_class, db_srid=None): + + if legacy_geoalchemy: + + package_extent_table = Table( + 'package_extent', meta.metadata, + Column('package_id', types.UnicodeText, primary_key=True), + GeometryExtensionColumn('the_geom', Geometry(2, srid=db_srid)) + ) + + meta.mapper( + package_extent_class, + package_extent_table, + properties={'the_geom': + GeometryColumn(package_extent_table.c.the_geom, + comparator=PGComparator)} + ) + + GeometryDDL(package_extent_table) + else: + + # PostGIS 1.5 requires management=True when defining the Geometry + # field + management = (postgis_version()[:1] == '1') + + package_extent_table = Table( + 'package_extent', meta.metadata, + Column('package_id', types.UnicodeText, primary_key=True), + Column('the_geom', Geometry('GEOMETRY', srid=db_srid, + management=management)), + ) + + meta.mapper(package_extent_class, package_extent_table) + + return package_extent_table + + +def compare_geometry_fields(geom_field1, geom_field2): + + return Session.scalar(ST_Equals(geom_field1, geom_field2)) diff --git a/ckanext/spatial/harvesters/gemini.py b/ckanext/spatial/harvesters/gemini.py index 52faff6..8dc65d4 100644 --- a/ckanext/spatial/harvesters/gemini.py +++ b/ckanext/spatial/harvesters/gemini.py @@ -76,7 +76,7 @@ class GeminiHarvester(SpatialHarvester): self._save_object_error('Error importing Gemini document.', harvest_object, 'Import') else: self._save_object_error('Error importing Gemini document: %s' % str(e), harvest_object, 'Import') - + raise if debug_exception_mode: raise @@ -356,10 +356,6 @@ class GeminiHarvester(SpatialHarvester): self.obj.current = True self.obj.save() - - assert gemini_guid == [e['value'] for e in package['extras'] if e['key'] == 'guid'][0] - assert self.obj.id == [e['value'] for e in package['extras'] if e['key'] == 'harvest_object_id'][0] - return package @classmethod diff --git a/ckanext/spatial/lib/__init__.py b/ckanext/spatial/lib/__init__.py index 05b0c79..5b0bb94 100644 --- a/ckanext/spatial/lib/__init__.py +++ b/ckanext/spatial/lib/__init__.py @@ -7,10 +7,13 @@ from ckan.lib.base import config from ckanext.spatial.model import PackageExtent from shapely.geometry import asShape -from geoalchemy import WKTSpatialElement +from ckanext.spatial.geoalchemy_common import (WKTElement, ST_Transform, + compare_geometry_fields, + ) log = logging.getLogger(__name__) + def get_srid(crs): """Returns the SRID for the provided CRS definition The CRS can be defined in the following formats @@ -52,7 +55,8 @@ def save_package_extent(package_id, geometry = None, srid = None): if not srid: srid = db_srid - package_extent = PackageExtent(package_id=package_id,the_geom=WKTSpatialElement(shape.wkt, srid)) + package_extent = PackageExtent(package_id=package_id, + the_geom=WKTElement(shape.wkt, srid)) # Check if extent exists if existing_package_extent: @@ -63,7 +67,7 @@ def save_package_extent(package_id, geometry = None, srid = None): log.debug('Deleted extent for package %s' % package_id) else: # Check if extent changed - if Session.scalar(package_extent.the_geom.wkt) <> Session.scalar(existing_package_extent.the_geom.wkt): + if not compare_geometry_fields(package_extent.the_geom, existing_package_extent.the_geom): # Update extent existing_package_extent.the_geom = package_extent.the_geom existing_package_extent.save() @@ -127,9 +131,9 @@ def _bbox_2_wkt(bbox, srid): if srid and srid != db_srid: # Input geometry needs to be transformed to the one used on the database - input_geometry = functions.transform(WKTSpatialElement(wkt,srid),db_srid) + input_geometry = ST_Transform(WKTElement(wkt,srid),db_srid) else: - input_geometry = WKTSpatialElement(wkt,db_srid) + input_geometry = WKTElement(wkt,db_srid) return input_geometry def bbox_query(bbox,srid=None): @@ -167,16 +171,16 @@ def bbox_query_ordered(bbox, srid=None): 'query_srid': input_geometry.srid} # First get the area of the query box - sql = "SELECT ST_Area(GeomFromText(:query_bbox, :query_srid));" + sql = "SELECT ST_Area(ST_GeomFromText(:query_bbox, :query_srid));" params['search_area'] = Session.execute(sql, params).fetchone()[0] # Uses spatial ranking method from "USGS - 2006-1279" (Lanfear) sql = """SELECT ST_AsBinary(package_extent.the_geom) AS package_extent_the_geom, - POWER(ST_Area(ST_Intersection(package_extent.the_geom, GeomFromText(:query_bbox, :query_srid))),2)/ST_Area(package_extent.the_geom)/:search_area as spatial_ranking, + POWER(ST_Area(ST_Intersection(package_extent.the_geom, ST_GeomFromText(:query_bbox, :query_srid))),2)/ST_Area(package_extent.the_geom)/:search_area as spatial_ranking, package_extent.package_id AS package_id FROM package_extent, package WHERE package_extent.package_id = package.id - AND ST_Intersects(package_extent.the_geom, GeomFromText(:query_bbox, :query_srid)) + AND ST_Intersects(package_extent.the_geom, ST_GeomFromText(:query_bbox, :query_srid)) AND package.state = 'active' ORDER BY spatial_ranking desc""" extents = Session.execute(sql, params).fetchall() diff --git a/ckanext/spatial/model/package_extent.py b/ckanext/spatial/model/package_extent.py index 908dcd8..e5a342c 100644 --- a/ckanext/spatial/model/package_extent.py +++ b/ckanext/spatial/model/package_extent.py @@ -1,10 +1,6 @@ from logging import getLogger -from sqlalchemy import types, Column, Table - -from geoalchemy import Geometry, GeometryColumn, GeometryDDL, GeometryExtensionColumn -from geoalchemy.postgis import PGComparator - +from sqlalchemy import Table from ckan.lib.base import config from ckan import model @@ -12,6 +8,8 @@ from ckan.model import Session from ckan.model import meta from ckan.model.domain_object import DomainObject +from ckanext.spatial.geoalchemy_common import setup_spatial_table + log = getLogger(__name__) package_extent_table = None @@ -58,6 +56,7 @@ class PackageExtent(DomainObject): self.package_id = package_id self.the_geom = the_geom + def define_spatial_tables(db_srid=None): global package_extent_table @@ -67,18 +66,4 @@ def define_spatial_tables(db_srid=None): else: db_srid = int(db_srid) - package_extent_table = Table('package_extent', meta.metadata, - Column('package_id', types.UnicodeText, primary_key=True), - GeometryExtensionColumn('the_geom', Geometry(2,srid=db_srid))) - - - meta.mapper(PackageExtent, package_extent_table, properties={ - 'the_geom': GeometryColumn(package_extent_table.c.the_geom, - comparator=PGComparator)}) - - # enable the DDL extension - GeometryDDL(package_extent_table) - - - - + package_extent_table = setup_spatial_table(PackageExtent, db_srid) diff --git a/ckanext/spatial/plugin.py b/ckanext/spatial/plugin.py index 7e62485..ffba73c 100644 --- a/ckanext/spatial/plugin.py +++ b/ckanext/spatial/plugin.py @@ -12,11 +12,39 @@ from ckan import plugins as p from ckan.lib.search import SearchError, PackageSearchQuery from ckan.lib.helpers import json -from ckanext.spatial.lib import save_package_extent,validate_bbox, bbox_query, bbox_query_ordered + +def check_geoalchemy_requirement(): + '''Checks if a suitable geoalchemy version installed + + Checks if geoalchemy2 is present when using CKAN >= 2.3, and raises + an ImportError otherwise so users can upgrade manually. + ''' + + msg = ('This version of ckanext-spatial requires {0}. ' + + 'Please install it by running `pip install {0}`.\n' + + 'For more details see the "Troubleshooting" section of the ' + + 'install documentation') + + if p.toolkit.check_ckan_version(min_version='2.3'): + try: + import geoalchemy2 + except ImportError: + raise ImportError(msg.format('geoalchemy2')) + else: + try: + import geoalchemy + except ImportError: + raise ImportError(msg.format('geoalchemy')) + +check_geoalchemy_requirement() + + +from ckanext.spatial.lib import save_package_extent, validate_bbox, bbox_query, bbox_query_ordered from ckanext.spatial.model.package_extent import setup as setup_model log = getLogger(__name__) + def package_error_summary(error_dict): ''' Do some i18n stuff on the error_dict keys ''' @@ -98,6 +126,7 @@ class SpatialMetadata(p.SingletonPlugin): error_dict = {'spatial':[u'Error creating geometry: %s' % str(e)]} raise p.toolkit.ValidationError(error_dict, error_summary=package_error_summary(error_dict)) except Exception, e: + raise if bool(os.getenv('DEBUG')): raise error_dict = {'spatial':[u'Error: %s' % str(e)]} diff --git a/ckanext/spatial/tests/base.py b/ckanext/spatial/tests/base.py index 81280de..dd4159c 100644 --- a/ckanext/spatial/tests/base.py +++ b/ckanext/spatial/tests/base.py @@ -5,27 +5,11 @@ from sqlalchemy import Table from nose.plugins.skip import SkipTest from ckan.model import Session, repo, meta, engine_is_sqlite -from ckanext.spatial.model.package_extent import setup as spatial_db_setup, define_spatial_tables +from ckanext.spatial.geoalchemy_common import postgis_version +from ckanext.spatial.model.package_extent import setup as spatial_db_setup from ckanext.harvest.model import setup as harvest_model_setup -def setup_postgis_tables(): - - conn = Session.connection() - script_path = os.path.join(os.path.dirname(os.path.abspath( __file__ )), 'scripts', 'postgis.sql') - script = open(script_path,'r').read() - for cmd in script.split(';'): - cmd = re.sub(r'--(.*)|[\n\t]','',cmd) - if len(cmd): - conn.execute(cmd) - - Session.commit() - - -class SpatialTestBase: - - db_srid = 4326 - - geojson_examples = { +geojson_examples = { 'point':'{"type":"Point","coordinates":[100.0,0.0]}', 'point_2':'{"type":"Point","coordinates":[20,10]}', 'line':'{"type":"LineString","coordinates":[[100.0,0.0],[101.0,1.0]]}', @@ -35,16 +19,52 @@ class SpatialTestBase: 'multiline':'{"type":"MultiLineString","coordinates":[[[100.0,0.0],[101.0,1.0]],[[102.0,2.0],[103.0,3.0]]]}', 'multipolygon':'{"type":"MultiPolygon","coordinates":[[[[102.0,2.0],[103.0,2.0],[103.0,3.0],[102.0,3.0],[102.0,2.0]]],[[[100.0,0.0],[101.0,0.0],[101.0,1.0],[100.0,1.0],[100.0,0.0]],[[100.2,0.2],[100.8,0.2],[100.8,0.8],[100.2,0.8],[100.2,0.2]]]]}'} + +def _execute_script(script_path): + + conn = Session.connection() + script = open(script_path, 'r').read() + for cmd in script.split(';'): + cmd = re.sub(r'--(.*)|[\n\t]', '', cmd) + if len(cmd): + conn.execute(cmd) + + Session.commit() + + +def create_postgis_tables(): + scripts_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'scripts') + if postgis_version()[:1] == '1': + _execute_script(os.path.join(scripts_path, 'spatial_ref_sys.sql')) + _execute_script(os.path.join(scripts_path, 'geometry_columns.sql')) + else: + _execute_script(os.path.join(scripts_path, 'spatial_ref_sys.sql')) + + +class SpatialTestBase(object): + + db_srid = 4326 + + geojson_examples = geojson_examples + @classmethod def setup_class(cls): if engine_is_sqlite(): raise SkipTest("PostGIS is required for this test") - + # This will create the PostGIS tables (geometry_columns and # spatial_ref_sys) which were deleted when rebuilding the database - table = Table('geometry_columns', meta.metadata) + table = Table('spatial_ref_sys', meta.metadata) if not table.exists(): - setup_postgis_tables() + create_postgis_tables() + + # When running the tests with the --reset-db option for some + # reason the metadata holds a reference to the `package_extent` + # table after being deleted, causing an InvalidRequestError + # exception when trying to recreate it further on + if 'package_extent' in meta.metadata.tables: + meta.metadata.remove(meta.metadata.tables['package_extent']) spatial_db_setup() diff --git a/ckanext/spatial/tests/functional/test_dataset_map.py b/ckanext/spatial/tests/functional/test_dataset_map.py deleted file mode 100644 index 1b59cb2..0000000 --- a/ckanext/spatial/tests/functional/test_dataset_map.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging -from pprint import pprint - -from ckan.model import Package, Session -from ckan import model -from ckan.lib.helpers import url_for,json - -from ckan.tests import CreateTestData -from ckan.tests.functional.base import FunctionalTestCase -from ckanext.spatial.tests.base import SpatialTestBase - -log = logging.getLogger(__name__) - - -class TestDatasetMap(FunctionalTestCase,SpatialTestBase): - def setup(self): - CreateTestData.create() - - def teardown(self): - model.repo.rebuild_db() - - def test_map_shown(self): - extra_environ = {'REMOTE_USER': 'annafan'} - name = 'annakarenina' - - offset = url_for(controller='package', action='edit',id=name) - res = self.app.get(offset, extra_environ=extra_environ) - assert 'Edit - Datasets' in res - fv = res.forms['dataset-edit'] - prefix = '' - fv[prefix+'extras__1__key'] = u'spatial' - fv[prefix+'extras__1__value'] = self.geojson_examples['point'] - - res = fv.submit('save', extra_environ=extra_environ) - assert not 'Error' in res, res - - # Load the dataset page and check if the libraries have been loaded - offset = url_for(controller='package', action='read',id=name) - res = self.app.get(offset) - - assert '
' in res, res - assert '' in res - assert self.geojson_examples['point'] in res diff --git a/ckanext/spatial/tests/functional/test_package.py b/ckanext/spatial/tests/functional/test_package.py index 3a230d7..58a7501 100644 --- a/ckanext/spatial/tests/functional/test_package.py +++ b/ckanext/spatial/tests/functional/test_package.py @@ -1,141 +1,143 @@ -import logging -from pprint import pprint +import json +from nose.tools import assert_equals -from ckan.model import Package, Session -from ckan.lib.helpers import url_for,json +from ckan.model import Session +from ckan.lib.helpers import url_for + +import ckan.new_tests.helpers as helpers +import ckan.new_tests.factories as factories -from ckan.tests import CreateTestData -from ckan.tests.functional.base import FunctionalTestCase from ckanext.spatial.model import PackageExtent - +from ckanext.spatial.geoalchemy_common import legacy_geoalchemy from ckanext.spatial.tests.base import SpatialTestBase -log = logging.getLogger(__name__) +class TestSpatialExtra(SpatialTestBase, helpers.FunctionalTestBase): -class TestPackageController(FunctionalTestCase,SpatialTestBase): + def test_spatial_extra(self): + app = self._get_test_app() - @classmethod - def setup_class(cls): - SpatialTestBase.setup_class() - cls.extra_environ = {'REMOTE_USER': 'annafan'} + user = factories.User() + env = {'REMOTE_USER': user['name'].encode('ascii')} + dataset = factories.Dataset(user=user) - def setup(self): - CreateTestData.create() + offset = url_for(controller='package', action='edit', id=dataset['id']) + res = app.get(offset, extra_environ=env) - def teardown(self): - CreateTestData.delete() + form = res.forms[1] + form['extras__0__key'] = u'spatial' + form['extras__0__value'] = self.geojson_examples['point'] - def test_new(self): - name = 'test-spatial-dataset-1' + res = helpers.submit_and_follow(app, form, env, 'save') - offset = url_for(controller='package', action='new') - res = self.app.get(offset, extra_environ=self.extra_environ) - assert 'Add - Datasets' in res - fv = res.forms['dataset-edit'] - prefix = '' - fv[prefix + 'name'] = name - fv[prefix+'extras__0__key'] = u'spatial' - fv[prefix+'extras__0__value'] = self.geojson_examples['point'] + assert 'Error' not in res, res - res = fv.submit('save', extra_environ=self.extra_environ) - assert not 'Error' in res, res - - package = Package.get(name) - - # Check that a PackageExtent object has been created - package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id==package.id).first() + package_extent = Session.query(PackageExtent) \ + .filter(PackageExtent.package_id == dataset['id']).first() geojson = json.loads(self.geojson_examples['point']) - assert package_extent - assert package_extent.package_id == package.id - assert Session.scalar(package_extent.the_geom.x) == geojson['coordinates'][0] - assert Session.scalar(package_extent.the_geom.y) == geojson['coordinates'][1] - assert Session.scalar(package_extent.the_geom.srid) == self.db_srid + assert_equals(package_extent.package_id, dataset['id']) + if legacy_geoalchemy: + assert_equals(Session.scalar(package_extent.the_geom.x), + geojson['coordinates'][0]) + assert_equals(Session.scalar(package_extent.the_geom.y), + geojson['coordinates'][1]) + assert_equals(Session.scalar(package_extent.the_geom.srid), + self.db_srid) + else: + from sqlalchemy import func + assert_equals( + Session.query(func.ST_X(package_extent.the_geom)).first()[0], + geojson['coordinates'][0]) + assert_equals( + Session.query(func.ST_Y(package_extent.the_geom)).first()[0], + geojson['coordinates'][1]) + assert_equals(package_extent.the_geom.srid, self.db_srid) - def test_new_bad_json(self): - name = 'test-spatial-dataset-2' + def test_spatial_extra_edit(self): + app = self._get_test_app() - offset = url_for(controller='package', action='new') - res = self.app.get(offset, extra_environ=self.extra_environ) - assert 'Add - Datasets' in res - fv = res.forms['dataset-edit'] - prefix = '' - fv[prefix + 'name'] = name - fv[prefix+'extras__0__key'] = u'spatial' - fv[prefix+'extras__0__value'] = u'{"Type":Bad Json]' + user = factories.User() + env = {'REMOTE_USER': user['name'].encode('ascii')} + dataset = factories.Dataset(user=user) + + offset = url_for(controller='package', action='edit', id=dataset['id']) + res = app.get(offset, extra_environ=env) + + form = res.forms[1] + form['extras__0__key'] = u'spatial' + form['extras__0__value'] = self.geojson_examples['point'] + + res = helpers.submit_and_follow(app, form, env, 'save') + + assert 'Error' not in res, res + + res = app.get(offset, extra_environ=env) + + form = res.forms[1] + form['extras__0__key'] = u'spatial' + form['extras__0__value'] = self.geojson_examples['polygon'] + + res = helpers.submit_and_follow(app, form, env, 'save') + + assert 'Error' not in res, res + + package_extent = Session.query(PackageExtent) \ + .filter(PackageExtent.package_id == dataset['id']).first() + + assert_equals(package_extent.package_id, dataset['id']) + if legacy_geoalchemy: + assert_equals( + Session.scalar(package_extent.the_geom.geometry_type), + 'ST_Polygon') + assert_equals( + Session.scalar(package_extent.the_geom.srid), + self.db_srid) + else: + from sqlalchemy import func + assert_equals( + Session.query( + func.ST_GeometryType(package_extent.the_geom)).first()[0], + 'ST_Polygon') + assert_equals(package_extent.the_geom.srid, self.db_srid) + + def test_spatial_extra_bad_json(self): + app = self._get_test_app() + + user = factories.User() + env = {'REMOTE_USER': user['name'].encode('ascii')} + dataset = factories.Dataset(user=user) + + offset = url_for(controller='package', action='edit', id=dataset['id']) + res = app.get(offset, extra_environ=env) + + form = res.forms[1] + form['extras__0__key'] = u'spatial' + form['extras__0__value'] = u'{"Type":Bad Json]' + + res = helpers.webtest_submit(form, extra_environ=env, name='save') - res = fv.submit('save', extra_environ=self.extra_environ) assert 'Error' in res, res assert 'Spatial' in res assert 'Error decoding JSON object' in res - # Check that package was not created - assert not Package.get(name) + def test_spatial_extra_bad_geojson(self): + app = self._get_test_app() - def test_new_bad_geojson(self): - name = 'test-spatial-dataset-3' + user = factories.User() + env = {'REMOTE_USER': user['name'].encode('ascii')} + dataset = factories.Dataset(user=user) - offset = url_for(controller='package', action='new') - res = self.app.get(offset, extra_environ=self.extra_environ) - assert 'Add - Datasets' in res - fv = res.forms['dataset-edit'] - prefix = '' - fv[prefix + 'name'] = name - fv[prefix+'extras__0__key'] = u'spatial' - fv[prefix+'extras__0__value'] = u'{"Type":"Bad_GeoJSON","a":2}' + offset = url_for(controller='package', action='edit', id=dataset['id']) + res = app.get(offset, extra_environ=env) + + form = res.forms[1] + form['extras__0__key'] = u'spatial' + form['extras__0__value'] = u'{"Type":"Bad_GeoJSON","a":2}' + + res = helpers.webtest_submit(form, extra_environ=env, name='save') - res = fv.submit('save', extra_environ=self.extra_environ) assert 'Error' in res, res assert 'Spatial' in res assert 'Error creating geometry' in res - - # Check that package was not created - assert not Package.get(name) - - def test_edit(self): - - name = 'annakarenina' - - offset = url_for(controller='package', action='edit',id=name) - res = self.app.get(offset, extra_environ=self.extra_environ) - assert 'Edit - Datasets' in res - fv = res.forms['dataset-edit'] - prefix = '' - fv[prefix+'extras__1__key'] = u'spatial' - fv[prefix+'extras__1__value'] = self.geojson_examples['point'] - - res = fv.submit('save', extra_environ=self.extra_environ) - assert not 'Error' in res, res - - package = Package.get(name) - - # Check that a PackageExtent object has been created - package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id==package.id).first() - geojson = json.loads(self.geojson_examples['point']) - - assert package_extent - assert package_extent.package_id == package.id - assert Session.scalar(package_extent.the_geom.x) == geojson['coordinates'][0] - assert Session.scalar(package_extent.the_geom.y) == geojson['coordinates'][1] - assert Session.scalar(package_extent.the_geom.srid) == self.db_srid - - # Update the spatial extra - offset = url_for(controller='package', action='edit',id=name) - res = self.app.get(offset, extra_environ=self.extra_environ) - assert 'Edit - Datasets' in res - fv = res.forms['dataset-edit'] - prefix = '' - fv[prefix+'extras__1__value'] = self.geojson_examples['polygon'] - - res = fv.submit('save', extra_environ=self.extra_environ) - assert not 'Error' in res, res - - # Check that the PackageExtent object has been updated - package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id==package.id).first() - assert package_extent - assert package_extent.package_id == package.id - assert Session.scalar(package_extent.the_geom.geometry_type) == 'ST_Polygon' - assert Session.scalar(package_extent.the_geom.srid) == self.db_srid - diff --git a/ckanext/spatial/tests/functional/test_spatial_query_widget.py b/ckanext/spatial/tests/functional/test_spatial_query_widget.py deleted file mode 100644 index 27365ca..0000000 --- a/ckanext/spatial/tests/functional/test_spatial_query_widget.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging -from pylons import config - -from ckan.lib.helpers import url_for - -from ckan.tests.functional.base import FunctionalTestCase - -from ckanext.spatial.tests.base import SpatialTestBase - -log = logging.getLogger(__name__) - - -class TestSpatialQueryWidget(FunctionalTestCase,SpatialTestBase): - - def test_widget_shown(self): - # Load the dataset search page and check if the libraries have been loaded - offset = url_for(controller='package', action='search') - res = self.app.get(offset) - - assert '
' in res, res - assert '' in res - assert config.get('ckan.spatial.default_extent') in res diff --git a/ckanext/spatial/tests/functional/test_widgets.py b/ckanext/spatial/tests/functional/test_widgets.py new file mode 100644 index 0000000..3029505 --- /dev/null +++ b/ckanext/spatial/tests/functional/test_widgets.py @@ -0,0 +1,34 @@ +from ckan.lib.helpers import url_for + +from ckanext.spatial.tests.base import SpatialTestBase + +import ckan.new_tests.helpers as helpers +import ckan.new_tests.factories as factories + + +class TestSpatialWidgets(SpatialTestBase, helpers.FunctionalTestBase): + + def test_dataset_map(self): + app = self._get_test_app() + + user = factories.User() + dataset = factories.Dataset( + user=user, + extras=[{'key': 'spatial', + 'value': self.geojson_examples['point']}] + ) + offset = url_for(controller='package', action='read', id=dataset['id']) + res = app.get(offset) + + assert 'data-module="dataset-map"' in res + assert 'dataset_map.js' in res + + def test_spatial_search_widget(self): + + app = self._get_test_app() + + offset = url_for(controller='package', action='search') + res = app.get(offset) + + assert 'data-module="spatial-query"' in res + assert 'spatial_query.js' in res diff --git a/ckanext/spatial/tests/functional/test_wms_preview.py b/ckanext/spatial/tests/functional/test_wms_preview.py deleted file mode 100644 index 2e2fe5d..0000000 --- a/ckanext/spatial/tests/functional/test_wms_preview.py +++ /dev/null @@ -1,65 +0,0 @@ -import logging -from pprint import pprint -from ckan import model -from ckan.model import Package, Resource -from ckan.lib.helpers import url_for,json - -from ckan.tests import CreateTestData -from ckan.tests.functional.base import FunctionalTestCase -from ckanext.harvest.model import setup as harvest_model_setup -from ckanext.spatial.tests.base import SpatialTestBase - -log = logging.getLogger(__name__) - - -class TestWMSPreview(FunctionalTestCase,SpatialTestBase): - - def setup(self): - CreateTestData.create() - - def teardown(self): - model.repo.rebuild_db() - - def test_link_and_map_shown(self): - from nose.plugins.skip import SkipTest - raise SkipTest('TODO: Need to update this to use logic functions') - - name = u'annakarenina' - - wms_url = 'http://maps.bgs.ac.uk/ArcGIS/services/BGS_Detailed_Geology/MapServer/WMSServer?' - rev = model.repo.new_revision() - pkg = Package.get(name) - pr = Resource(url=wms_url,format='WMS') - pkg.resources.append(pr) - pkg.save() - model.repo.commit_and_remove() - # Load the dataset page and check if link appears - offset = url_for(controller='package', action='read',id=name) - res = self.app.get(offset) - - assert 'View available WMS layers' in res, res - - # Load the dataset map preview page and check if libraries are loaded - offset = '/dataset/%s/map' % name - res = self.app.get(offset) - assert '' in res, res - assert 'CKAN.WMSPreview.setup("%s");' % wms_url.split('?')[0] in res - - def test_link_and_map_not_shown(self): - - name = 'annakarenina' - - offset = url_for(controller='package', action='read',id=name) - - # Load the dataset page and check that link does not appear - offset = url_for(controller='package', action='read',id=name) - res = self.app.get(offset) - - assert not 'View available WMS layers' in res, res - - # Load the dataset map preview page and check that libraries are not loaded - offset = '/dataset/%s/map' % name - res = self.app.get(offset, status=400) - assert '400 Bad Request' in res, res - assert 'This dataset does not have a WMS resource' in res - assert not '' in res diff --git a/ckanext/spatial/tests/lib/test_spatial.py b/ckanext/spatial/tests/lib/test_spatial.py index 21971c6..2e46aef 100644 --- a/ckanext/spatial/tests/lib/test_spatial.py +++ b/ckanext/spatial/tests/lib/test_spatial.py @@ -3,15 +3,45 @@ import random from nose.tools import assert_equal +from shapely.geometry import asShape + from ckan import model from ckan import plugins from ckan.lib.helpers import json -from ckan.logic.schema import default_create_package_schema from ckan.logic.action.create import package_create from ckan.lib.munge import munge_title_to_name + +from ckanext.spatial.model import PackageExtent from ckanext.spatial.lib import validate_bbox, bbox_query, bbox_query_ordered +from ckanext.spatial.geoalchemy_common import WKTElement, compare_geometry_fields from ckanext.spatial.tests.base import SpatialTestBase + +class TestCompareGeometries(SpatialTestBase): + + def _get_extent_object(self, geometry): + if isinstance(geometry, basestring): + geometry = json.loads(geometry) + shape = asShape(geometry) + return PackageExtent(package_id='xxx', + the_geom=WKTElement(shape.wkt, 4326)) + + def test_same_points(self): + + extent1 = self._get_extent_object(self.geojson_examples['point']) + extent2 = self._get_extent_object(self.geojson_examples['point']) + + assert compare_geometry_fields(extent1.the_geom, extent2.the_geom) + + def test_different_points(self): + + extent1 = self._get_extent_object(self.geojson_examples['point']) + extent2 = self._get_extent_object(self.geojson_examples['point_2']) + + assert not compare_geometry_fields(extent1.the_geom, extent2.the_geom) + + + class TestValidateBbox: bbox_dict = {'minx': -4.96, 'miny': 55.70, diff --git a/ckanext/spatial/tests/model/test_package_extent.py b/ckanext/spatial/tests/model/test_package_extent.py index 6bec72a..249c1a5 100644 --- a/ckanext/spatial/tests/model/test_package_extent.py +++ b/ckanext/spatial/tests/model/test_package_extent.py @@ -1,61 +1,87 @@ -import logging -from pprint import pprint - -from geoalchemy import WKTSpatialElement - +from nose.tools import assert_equals from shapely.geometry import asShape -from ckan.model import Session, Package -from ckan import model + +from ckan.model import Session from ckan.lib.helpers import json -from ckan.tests import CreateTestData +from ckan.new_tests import factories + from ckanext.spatial.model import PackageExtent - +from ckanext.spatial.geoalchemy_common import WKTElement, legacy_geoalchemy from ckanext.spatial.tests.base import SpatialTestBase -log = logging.getLogger(__name__) - - class TestPackageExtent(SpatialTestBase): - def setup(self): - CreateTestData.create() - - def teardown(self): - model.repo.rebuild_db() def test_create_extent(self): - package = Package.get('annakarenina') - assert package + + package = factories.Dataset() geojson = json.loads(self.geojson_examples['point']) shape = asShape(geojson) - package_extent = PackageExtent(package_id=package.id,the_geom=WKTSpatialElement(shape.wkt, self.db_srid)) + package_extent = PackageExtent(package_id=package['id'], + the_geom=WKTElement(shape.wkt, + self.db_srid)) package_extent.save() - assert package_extent.package_id == package.id - assert Session.scalar(package_extent.the_geom.x) == geojson['coordinates'][0] - assert Session.scalar(package_extent.the_geom.y) == geojson['coordinates'][1] - assert Session.scalar(package_extent.the_geom.srid) == self.db_srid + assert_equals(package_extent.package_id, package['id']) + if legacy_geoalchemy: + assert_equals(Session.scalar(package_extent.the_geom.x), + geojson['coordinates'][0]) + assert_equals(Session.scalar(package_extent.the_geom.y), + geojson['coordinates'][1]) + assert_equals(Session.scalar(package_extent.the_geom.srid), + self.db_srid) + else: + from sqlalchemy import func + assert_equals( + Session.query(func.ST_X(package_extent.the_geom)).first()[0], + geojson['coordinates'][0]) + assert_equals( + Session.query(func.ST_Y(package_extent.the_geom)).first()[0], + geojson['coordinates'][1]) + assert_equals(package_extent.the_geom.srid, self.db_srid) def test_update_extent(self): - package = Package.get('annakarenina') + package = factories.Dataset() geojson = json.loads(self.geojson_examples['point']) shape = asShape(geojson) - package_extent = PackageExtent(package_id=package.id,the_geom=WKTSpatialElement(shape.wkt, self.db_srid)) + package_extent = PackageExtent(package_id=package['id'], + the_geom=WKTElement(shape.wkt, + self.db_srid)) package_extent.save() - assert Session.scalar(package_extent.the_geom.geometry_type) == 'ST_Point' + if legacy_geoalchemy: + assert_equals( + Session.scalar(package_extent.the_geom.geometry_type), + 'ST_Point') + else: + from sqlalchemy import func + assert_equals( + Session.query( + func.ST_GeometryType(package_extent.the_geom)).first()[0], + 'ST_Point') # Update the geometry (Point -> Polygon) geojson = json.loads(self.geojson_examples['polygon']) shape = asShape(geojson) - package_extent.the_geom=WKTSpatialElement(shape.wkt, self.db_srid) + package_extent.the_geom = WKTElement(shape.wkt, self.db_srid) package_extent.save() - assert package_extent.package_id == package.id - assert Session.scalar(package_extent.the_geom.geometry_type) == 'ST_Polygon' - assert Session.scalar(package_extent.the_geom.srid) == self.db_srid + assert_equals(package_extent.package_id, package['id']) + if legacy_geoalchemy: + assert_equals( + Session.scalar(package_extent.the_geom.geometry_type), + 'ST_Polygon') + assert_equals( + Session.scalar(package_extent.the_geom.srid), + self.db_srid) + else: + assert_equals( + Session.query( + func.ST_GeometryType(package_extent.the_geom)).first()[0], + 'ST_Polygon') + assert_equals(package_extent.the_geom.srid, self.db_srid) diff --git a/ckanext/spatial/tests/scripts/geometry_columns.sql b/ckanext/spatial/tests/scripts/geometry_columns.sql new file mode 100644 index 0000000..e2bbb75 --- /dev/null +++ b/ckanext/spatial/tests/scripts/geometry_columns.sql @@ -0,0 +1,25 @@ +------------------------------------------------------------------- +-- WARNING: This is probably NOT the file you are looking for. +-- This file is intended to be used only during tests, you won't +-- get a functional PostGIS database executing it. Please install +-- PostGIS as described in the README. +------------------------------------------------------------------- + +------------------------------------------------------------------- +-- GEOMETRY_COLUMNS +------------------------------------------------------------------- +CREATE TABLE geometry_columns ( + f_table_catalog varchar(256) not null, + f_table_schema varchar(256) not null, + f_table_name varchar(256) not null, + f_geometry_column varchar(256) not null, + coord_dimension integer not null, + srid integer not null, + type varchar(30) not null, + CONSTRAINT geometry_columns_pk primary key ( + f_table_catalog, + f_table_schema, + f_table_name, + f_geometry_column ) +) WITH OIDS; + diff --git a/ckanext/spatial/tests/scripts/spatial_ref_sys.sql b/ckanext/spatial/tests/scripts/spatial_ref_sys.sql new file mode 100644 index 0000000..467a868 --- /dev/null +++ b/ckanext/spatial/tests/scripts/spatial_ref_sys.sql @@ -0,0 +1,23 @@ +------------------------------------------------------------------- +-- WARNING: This is probably NOT the file you are looking for. +-- This file is intended to be used only during tests, you won't +-- get a functional PostGIS database executing it. Please install +-- PostGIS as described in the README. +------------------------------------------------------------------- + +------------------------------------------------------------------- +-- SPATIAL_REF_SYS +------------------------------------------------------------------- +CREATE TABLE spatial_ref_sys ( + srid integer not null primary key, + auth_name varchar(256), + auth_srid integer, + srtext varchar(2048), + proj4text varchar(2048) +); + +--- +--- EPSG 4326 : WGS 84 +--- +INSERT INTO "spatial_ref_sys" ("srid","auth_name","auth_srid","srtext","proj4text") VALUES (4326,'EPSG',4326,'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]','+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs '); + diff --git a/ckanext/spatial/tests/test_api.py b/ckanext/spatial/tests/test_api.py index b21e72b..5358650 100644 --- a/ckanext/spatial/tests/test_api.py +++ b/ckanext/spatial/tests/test_api.py @@ -1,217 +1,147 @@ -import logging -import json -from pprint import pprint from nose.plugins.skip import SkipTest -from nose.tools import assert_equal, assert_raises -from ckan.logic.action.create import package_create -from ckan.logic.action.delete import package_delete -from ckan.logic.schema import default_create_package_schema -from ckan import model +from nose.tools import assert_equals, assert_raises + +from ckan.model import Session +from ckan.lib.search import SearchError +import ckan.new_tests.helpers as helpers +import ckan.new_tests.factories as factories -from ckan.model import Package, Session -import ckan.lib.search as search -from ckan.tests import CreateTestData, setup_test_search_index,WsgiAppCase -from ckan.tests.functional.api.base import ApiTestCase -from ckan.tests import TestController as ControllerTestCase from ckanext.spatial.tests.base import SpatialTestBase -log = logging.getLogger(__name__) + +class TestAction(SpatialTestBase): + + def test_spatial_query(self): + + dataset = factories.Dataset( + extras=[{'key': 'spatial', + 'value': self.geojson_examples['point']}] + ) + + result = helpers.call_action( + 'package_search', + extras={'ext_bbox': '-180,-90,180,90'}) + + assert_equals(result['count'], 1) + assert_equals(result['results'][0]['title'], dataset['title']) + + def test_spatial_query_outside_bbox(self): + + factories.Dataset( + extras=[{'key': 'spatial', + 'value': self.geojson_examples['point']}] + ) + + result = helpers.call_action( + 'package_search', + extras={'ext_bbox': '-10,-20,10,20'}) + + assert_equals(result['count'], 0) + + def test_spatial_query_wrong_bbox(self): + + assert_raises(SearchError, helpers.call_action, + 'package_search', extras={'ext_bbox': '-10,-20,10,a'}) +class TestHarvestedMetadataAPI(SpatialTestBase, helpers.FunctionalTestBase): -class TestSpatialApi(ApiTestCase,SpatialTestBase,ControllerTestCase): - - api_version = '2' - - @classmethod - def setup_class(self): - super(TestSpatialApi,self).setup_class() - setup_test_search_index() - CreateTestData.create_test_user() - self.package_fixture_data = { - 'name' : u'test-spatial-dataset-search-point', - 'title': 'Some Title', - 'extras': [{'key':'spatial','value':self.geojson_examples['point']}] - } - self.base_url = self.offset('/search/dataset/geo') - - def _offset_with_bbox(self,minx=-180,miny=-90,maxx=180,maxy=90,crs=None): - offset = self.base_url + '?bbox=%s,%s,%s,%s' % (minx,miny,maxx,maxy) - if crs: - offset = offset + '&crs=%s' % crs - return offset - - def test_basic_query(self): - schema = default_create_package_schema() - context = {'model':model,'session':Session,'user':'tester','extras_as_string':True,'schema':schema,'api_version':2} - package_dict = package_create(context,self.package_fixture_data) - package_id = context.get('id') - - # Point inside bbox - offset = self._offset_with_bbox() - - res = self.app.get(offset, status=200) - res_dict = self.data_from_res(res) - - assert res_dict['count'] == 1 - assert res_dict['results'][0] == package_id - - # Point outside bbox - offset = self._offset_with_bbox(-10,10,-20,20) - - res = self.app.get(offset, status=200) - res_dict = self.data_from_res(res) - - assert res_dict['count'] == 0 - assert res_dict['results'] == [] - - # Delete the package and ensure it does not come up on - # search results - package_delete(context,{'id':package_id}) - offset = self._offset_with_bbox() - - res = self.app.get(offset, status=200) - res_dict = self.data_from_res(res) - - assert res_dict['count'] == 0 - assert res_dict['results'] == [] - - - -class TestActionPackageSearch(SpatialTestBase,WsgiAppCase): - - @classmethod - def setup_class(self): - super(TestActionPackageSearch,self).setup_class() - setup_test_search_index() - self.package_fixture_data_1 = { - 'name' : u'test-spatial-dataset-search-point-1', - 'title': 'Some Title 1', - 'extras': [{'key':'spatial','value':self.geojson_examples['point']}] - } - self.package_fixture_data_2 = { - 'name' : u'test-spatial-dataset-search-point-2', - 'title': 'Some Title 2', - 'extras': [{'key':'spatial','value':self.geojson_examples['point_2']}] - } - - CreateTestData.create() - - @classmethod - def teardown_class(self): - model.repo.rebuild_db() - - def test_1_basic(self): - schema = default_create_package_schema() - context = {'model':model,'session':Session,'user':'tester','extras_as_string':True,'schema':schema,'api_version':2} - package_dict_1 = package_create(context,self.package_fixture_data_1) - del context['package'] - package_dict_2 = package_create(context,self.package_fixture_data_2) - - postparams = '%s=1' % json.dumps({ - 'q': 'test', - 'facet.field': ('groups', 'tags', 'res_format', 'license'), - 'rows': 20, - 'start': 0, - 'extras': { - 'ext_bbox': '%s,%s,%s,%s' % (10,10,40,40) - } - }) - res = self.app.post('/api/action/package_search', params=postparams) - res = json.loads(res.body) - result = res['result'] - - # Only one dataset returned - assert_equal(res['success'], True) - assert_equal(result['count'], 1) - assert_equal(result['results'][0]['name'], 'test-spatial-dataset-search-point-2') - - -class TestHarvestedMetadataAPI(WsgiAppCase): - - - @classmethod - def setup_class(cls): + def test_api(self): try: - from ckanext.harvest.model import HarvestObject, HarvestJob, HarvestSource, HarvestObjectExtra + from ckanext.harvest.model import (HarvestObject, HarvestJob, + HarvestSource, + HarvestObjectExtra) except ImportError: raise SkipTest('The harvester extension is needed for these tests') - cls.content1 = 'Content 1' - ho1 = HarvestObject(guid='test-ho-1', - job=HarvestJob(source=HarvestSource(url='http://', type='xx')), - content=cls.content1) + content1 = 'Content 1' + ho1 = HarvestObject( + guid='test-ho-1', + job=HarvestJob(source=HarvestSource(url='http://', type='xx')), + content=content1) - cls.content2 = 'Content 2' - cls.original_content2 = 'Original Content 2' - ho2 = HarvestObject(guid='test-ho-2', - job=HarvestJob(source=HarvestSource(url='http://', type='xx')), - content=cls.content2) + content2 = 'Content 2' + original_content2 = 'Original Content 2' + ho2 = HarvestObject( + guid='test-ho-2', + job=HarvestJob(source=HarvestSource(url='http://', type='xx')), + content=content2) - hoe = HarvestObjectExtra(key='original_document', - value=cls.original_content2, - object=ho2) + hoe = HarvestObjectExtra( + key='original_document', + value=original_content2, + object=ho2) Session.add(ho1) Session.add(ho2) Session.add(hoe) Session.commit() - cls.object_id_1 = ho1.id - cls.object_id_2 = ho2.id + object_id_1 = ho1.id + object_id_2 = ho2.id - - def test_api(self): + app = self._get_test_app() # Test redirects for old URLs - url = '/api/2/rest/harvestobject/{0}/xml'.format(self.object_id_1) - r = self.app.get(url) - assert r.status == 301 - assert '/harvest/object/{0}'.format(self.object_id_1) in r.header_dict['Location'] - - url = '/api/2/rest/harvestobject/{0}/html'.format(self.object_id_1) - r = self.app.get(url) - assert r.status == 301 - assert '/harvest/object/{0}/html'.format(self.object_id_1) in r.header_dict['Location'] + url = '/api/2/rest/harvestobject/{0}/xml'.format(object_id_1) + r = app.get(url) + assert_equals(r.status_int, 301) + assert ('/harvest/object/{0}'.format(object_id_1) + in r.headers['Location']) + url = '/api/2/rest/harvestobject/{0}/html'.format(object_id_1) + r = app.get(url) + assert_equals(r.status_int, 301) + assert ('/harvest/object/{0}/html'.format(object_id_1) + in r.headers['Location']) # Access object content - url = '/harvest/object/{0}'.format(self.object_id_1) - r = self.app.get(url) - assert r.status == 200 - assert r.header_dict['Content-Type'] == 'application/xml; charset=utf-8' - assert r.body == self.content1 + url = '/harvest/object/{0}'.format(object_id_1) + r = app.get(url) + assert_equals(r.status_int, 200) + assert_equals(r.headers['Content-Type'], + 'application/xml; charset=utf-8') + assert_equals( + r.body, + '\nContent 1') # Access original content in object extra (if present) - url = '/harvest/object/{0}/original'.format(self.object_id_1) - r = self.app.get(url, status=404) - assert r.status == 404 + url = '/harvest/object/{0}/original'.format(object_id_1) + r = app.get(url, status=404) + assert_equals(r.status_int, 404) - url = '/harvest/object/{0}/original'.format(self.object_id_2) - r = self.app.get(url) - assert r.status == 200 - assert r.header_dict['Content-Type'] == 'application/xml; charset=utf-8' - assert r.body == self.original_content2 + url = '/harvest/object/{0}/original'.format(object_id_2) + r = app.get(url) + assert_equals(r.status_int, 200) + assert_equals(r.headers['Content-Type'], + 'application/xml; charset=utf-8') + assert_equals( + r.body, + '\n' + + 'Original Content 2') # Access HTML transformation - url = '/harvest/object/{0}/html'.format(self.object_id_1) - r = self.app.get(url) - assert r.status == 200 - assert r.header_dict['Content-Type'] == 'text/html; charset=utf-8' + url = '/harvest/object/{0}/html'.format(object_id_1) + r = app.get(url) + assert_equals(r.status_int, 200) + assert_equals(r.headers['Content-Type'], + 'text/html; charset=utf-8') assert 'GEMINI record about' in r.body - url = '/harvest/object/{0}/html/original'.format(self.object_id_1) - r = self.app.get(url, status=404) - assert r.status == 404 + url = '/harvest/object/{0}/html/original'.format(object_id_1) + r = app.get(url, status=404) + assert_equals(r.status_int, 404) - url = '/harvest/object/{0}/html'.format(self.object_id_2) - r = self.app.get(url) - assert r.status == 200 - assert r.header_dict['Content-Type'] == 'text/html; charset=utf-8' + url = '/harvest/object/{0}/html'.format(object_id_2) + r = app.get(url) + assert_equals(r.status_int, 200) + assert_equals(r.headers['Content-Type'], + 'text/html; charset=utf-8') assert 'GEMINI record about' in r.body - url = '/harvest/object/{0}/html/original'.format(self.object_id_2) - r = self.app.get(url) - assert r.status == 200 - assert r.header_dict['Content-Type'] == 'text/html; charset=utf-8' + url = '/harvest/object/{0}/html/original'.format(object_id_2) + r = app.get(url) + assert_equals(r.status_int, 200) + assert_equals(r.headers['Content-Type'], + 'text/html; charset=utf-8') assert 'GEMINI record about' in r.body diff --git a/ckanext/spatial/tests/test_csw_client.py b/ckanext/spatial/tests/test_csw_client.py index 5a6ef85..fee1a15 100644 --- a/ckanext/spatial/tests/test_csw_client.py +++ b/ckanext/spatial/tests/test_csw_client.py @@ -9,7 +9,6 @@ from owslib.iso import MD_Metadata from pylons import config from nose.plugins.skip import SkipTest -#from ckan.tests import CkanServerCase from ckan.model import engine_is_sqlite service = "http://ogcdev.bgs.ac.uk/geonetwork/srv/en/csw" diff --git a/ckanext/spatial/tests/test_harvest.py b/ckanext/spatial/tests/test_harvest.py index 3deda2a..6c64089 100644 --- a/ckanext/spatial/tests/test_harvest.py +++ b/ckanext/spatial/tests/test_harvest.py @@ -3,20 +3,17 @@ import lxml from nose.plugins.skip import SkipTest from nose.tools import assert_equal, assert_in, assert_raises -from ckan import plugins from ckan.lib.base import config from ckan import model -from ckan.model import Session,Package +from ckan.model import Session, Package from ckan.logic.schema import default_update_package_schema from ckan.logic import get_action -from ckanext.harvest.model import (setup as harvest_model_setup, - HarvestSource, HarvestJob, HarvestObject) -from ckanext.spatial.validation import Validators, SchematronValidator +from ckanext.harvest.model import (HarvestSource, HarvestJob, HarvestObject) +from ckanext.spatial.validation import Validators from ckanext.spatial.harvesters.gemini import (GeminiDocHarvester, - GeminiWafHarvester, - GeminiHarvester) + GeminiWafHarvester, + GeminiHarvester) from ckanext.spatial.harvesters.base import SpatialHarvester -from ckanext.spatial.model.package_extent import setup as spatial_db_setup from ckanext.spatial.tests.base import SpatialTestBase from xml_file_server import serve @@ -24,11 +21,8 @@ from xml_file_server import serve # Start simple HTTP server that serves XML test files serve() -class HarvestFixtureBase(SpatialTestBase): - @classmethod - def setup_class(cls): - SpatialTestBase.setup_class() +class HarvestFixtureBase(SpatialTestBase): def setup(self): # Add sysadmin user @@ -117,8 +111,8 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1-waf/index.html', 'source_type': u'gemini-waf' } @@ -142,7 +136,7 @@ class TestHarvest(HarvestFixtureBase): objects.append(obj) harvester.import_stage(obj) - pkgs = Session.query(Package).filter(Package.type!=u'harvest_source').all() + pkgs = Session.query(Package).filter(Package.type!=u'harvest').all() assert_equal(len(pkgs), 2) @@ -156,8 +150,8 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/service1.xml', 'source_type': u'gemini-single' } @@ -218,7 +212,7 @@ class TestHarvest(HarvestFixtureBase): 'bbox-north-lat': u'61.0243', 'bbox-south-lat': u'54.4764484375', 'bbox-west-long': u'-9.099786875', - 'spatial': u'{"type": "Polygon", "coordinates": [[[0.5242365625, 54.4764484375], [0.5242365625, 61.0243], [-9.099786875, 61.0243], [-9.099786875, 54.4764484375], [0.5242365625, 54.4764484375]]]}', + 'spatial': u'{"type": "Polygon", "coordinates": [[[0.5242365625, 54.4764484375], [-9.099786875, 54.4764484375], [-9.099786875, 61.0243], [0.5242365625, 61.0243], [0.5242365625, 54.4764484375]]]}', # Other 'coupled-resource': u'[{"href": ["http://scotgovsdi.edina.ac.uk/srv/en/csw?service=CSW&request=GetRecordById&version=2.0.2&outputSchema=http://www.isotc211.org/2005/gmd&elementSetName=full&id=250ea276-48e2-4189-8a89-fcc4ca92d652"], "uuid": ["250ea276-48e2-4189-8a89-fcc4ca92d652"], "title": []}]', 'dataset-reference-date': u'[{"type": "publication", "value": "2011-09-08"}]', @@ -244,7 +238,6 @@ class TestHarvest(HarvestFixtureBase): expected_resource = { 'ckan_recommended_wms_preview': 'True', 'description': 'Link to the GetCapabilities request for this service', - 'format': 'wms', # Newer CKAN versions lower case resource formats 'name': 'Web Map Service (WMS)', 'resource_locator_function': 'download', 'resource_locator_protocol': 'OGC:WMS-1.3.0-http-get-capabilities', @@ -260,13 +253,14 @@ class TestHarvest(HarvestFixtureBase): raise AssertionError('Unexpected value in resource for %s: %s (was expecting %s)' % \ (key, resource[key], value)) assert datetime.strptime(resource['verified_date'],'%Y-%m-%dT%H:%M:%S.%f').date() == date.today() + assert resource['format'].lower() == 'wms' def test_harvest_fields_dataset(self): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml', 'source_type': u'gemini-single' } @@ -326,7 +320,7 @@ class TestHarvest(HarvestFixtureBase): 'bbox-north-lat': u'61.06066944', 'bbox-south-lat': u'54.529947158', 'bbox-west-long': u'-8.97114288', - 'spatial': u'{"type": "Polygon", "coordinates": [[[0.205857204, 54.529947158], [0.205857204, 61.06066944], [-8.97114288, 61.06066944], [-8.97114288, 54.529947158], [0.205857204, 54.529947158]]]}', + 'spatial': u'{"type": "Polygon", "coordinates": [[[0.205857204, 54.529947158], [-8.97114288, 54.529947158], [-8.97114288, 61.06066944], [0.205857204, 61.06066944], [0.205857204, 54.529947158]]]}', # Other 'coupled-resource': u'[]', 'dataset-reference-date': u'[{"type": "creation", "value": "2004-02"}, {"type": "revision", "value": "2006-07-03"}]', @@ -368,8 +362,8 @@ class TestHarvest(HarvestFixtureBase): def test_harvest_error_bad_xml(self): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/error_bad_xml.xml', 'source_type': u'gemini-single' } @@ -394,8 +388,8 @@ class TestHarvest(HarvestFixtureBase): def test_harvest_error_404(self): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/not_there.xml', 'source_type': u'gemini-single' } @@ -416,8 +410,8 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/error_validation.xml', 'source_type': u'gemini-single' } @@ -459,8 +453,8 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml', 'source_type': u'gemini-single' } @@ -525,8 +519,8 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/service1.xml', 'source_type': u'gemini-single' } @@ -598,8 +592,8 @@ class TestHarvest(HarvestFixtureBase): # Create source1 source1_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/source1/same_dataset.xml', 'source_type': u'gemini-single' } @@ -620,8 +614,8 @@ class TestHarvest(HarvestFixtureBase): # (As of https://github.com/okfn/ckanext-inspire/commit/9fb67 # we are no longer throwing an exception when this happens) source2_fixture = { - 'title': 'Test Source 2', - 'name': 'test-source-2', + 'title': 'Test Source 2', + 'name': 'test-source-2', 'url': u'http://127.0.0.1:8999/gemini2.1/source2/same_dataset.xml', 'source_type': u'gemini-single' } @@ -667,8 +661,8 @@ class TestHarvest(HarvestFixtureBase): # Create source1 source1_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/source1/same_dataset.xml', 'source_type': u'gemini-single' } @@ -690,8 +684,8 @@ class TestHarvest(HarvestFixtureBase): # Harvest the same document, unchanged, from another source source2_fixture = { - 'title': 'Test Source 2', - 'name': 'test-source-2', + 'title': 'Test Source 2', + 'name': 'test-source-2', 'url': u'http://127.0.0.1:8999/gemini2.1/source2/same_dataset.xml', 'source_type': u'gemini-single' } @@ -714,8 +708,8 @@ class TestHarvest(HarvestFixtureBase): # Create source1 source1_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/service1.xml', 'source_type': u'gemini-single' } @@ -733,8 +727,8 @@ class TestHarvest(HarvestFixtureBase): # Harvest the same document GUID but with a newer date, from another source. source2_fixture = { - 'title': 'Test Source 2', - 'name': 'test-source-2', + 'title': 'Test Source 2', + 'name': 'test-source-2', 'url': u'http://127.0.0.1:8999/gemini2.1/service1_newer.xml', 'source_type': u'gemini-single' } @@ -760,8 +754,8 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml', 'source_type': u'gemini-single' } @@ -823,8 +817,8 @@ class TestGatherMethods(HarvestFixtureBase): HarvestFixtureBase.setup(self) # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml', 'source_type': u'gemini-single' } @@ -941,6 +935,7 @@ class TestImportStageTools: assert_equal(GeminiHarvester._process_responsible_organisation(responsible_organisation), ('', [])) + class TestValidation(HarvestFixtureBase): @classmethod @@ -955,8 +950,8 @@ class TestValidation(HarvestFixtureBase): def get_validation_errors(self, validation_test_filename): # Create source source_fixture = { - 'title': 'Test Source', - 'name': 'test-source', + 'title': 'Test Source', + 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/validation/%s' % validation_test_filename, 'source_type': u'gemini-single' } diff --git a/ckanext/spatial/tests/test_plugin/__init__.py b/ckanext/spatial/tests/test_plugin/__init__.py new file mode 100644 index 0000000..2e2033b --- /dev/null +++ b/ckanext/spatial/tests/test_plugin/__init__.py @@ -0,0 +1,7 @@ +# this is a namespace package +try: + import pkg_resources + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/ckanext/spatial/tests/test_plugin/plugin.py b/ckanext/spatial/tests/test_plugin/plugin.py new file mode 100644 index 0000000..2aa5a3d --- /dev/null +++ b/ckanext/spatial/tests/test_plugin/plugin.py @@ -0,0 +1,9 @@ +from ckan import plugins as p + + +class TestSpatialPlugin(p.SingletonPlugin): + + p.implements(p.IConfigurer, inherit=True) + + def update_config(self, config): + p.toolkit.add_template_directory(config, 'templates') diff --git a/ckanext/spatial/tests/test_plugin/templates/package/read_base.html b/ckanext/spatial/tests/test_plugin/templates/package/read_base.html new file mode 100644 index 0000000..02a5cb6 --- /dev/null +++ b/ckanext/spatial/tests/test_plugin/templates/package/read_base.html @@ -0,0 +1,11 @@ +{% ckan_extends %} + +{% block secondary_content %} + {{ super() }} + + {% set dataset_extent = h.get_pkg_dict_extra(c.pkg_dict, 'spatial', '') %} + {% if dataset_extent %} + {% snippet "spatial/snippets/dataset_map_sidebar.html", extent=dataset_extent %} + {% endif %} + +{% endblock %} diff --git a/ckanext/spatial/tests/test_plugin/templates/package/search.html b/ckanext/spatial/tests/test_plugin/templates/package/search.html new file mode 100644 index 0000000..2d80283 --- /dev/null +++ b/ckanext/spatial/tests/test_plugin/templates/package/search.html @@ -0,0 +1,9 @@ +{% ckan_extends %} + +{% block secondary_content %} + + {% snippet "spatial/snippets/spatial_query.html" %} + + {{ super() }} + +{% endblock %} diff --git a/doc/install.rst b/doc/install.rst index d3ab880..b77532f 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -13,13 +13,50 @@ Install PostGIS and system packages install any of the packages on this section and can skip to the next one. -.. note:: The package names and paths shown are the defaults on an Ubuntu - 12.04 install (PostgreSQL 9.1 and PostGIS 1.5). Adjust the - package names and the paths if you are using a different version of - any of them. +.. note:: The package names and paths shown are the defaults on Ubuntu installs. + Adjust the package names and the paths if you are using a different platform. All commands assume an existing CKAN database named ``ckan_default``. +Ubuntu 14.04 (PostgreSQL 9.3 and PostGIS 2.1) ++++++++++++++++++++++++++++++++++++++++++++++ + +#. Install PostGIS:: + + sudo apt-get install postgresql-9.3-postgis + +#. Run the following commands. The first one will create the necessary + tables and functions in the database, and the second will populate + the spatial reference table:: + + sudo -u postgres psql -d ckan_default -f /usr/share/postgresql/9.3/contrib/postgis-2.1/postgis.sql + sudo -u postgres psql -d ckan_default -f /usr/share/postgresql/9.3/contrib/postgis-2.1/spatial_ref_sys.sql + +#. Change the owner to spatial tables to the CKAN user to avoid errors later + on:: + + sudo -u postgres psql -d ckan_default -c 'ALTER VIEW geometry_columns OWNER TO ckan_default;' + sudo -u postgres psql -d ckan_default -c 'ALTER TABLE spatial_ref_sys OWNER TO ckan_default;' + +#. Execute the following command to see if PostGIS was properly + installed:: + + sudo -u postgres psql -d ckan_default -c "SELECT postgis_full_version()" + + You should get something like:: + + postgis_full_version + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + POSTGIS="2.1.2 r12389" GEOS="3.4.2-CAPI-1.8.2 r3921" PROJ="Rel. 4.8.0, 6 March 2012" GDAL="GDAL 1.10.1, released 2013/08/26" LIBXML="2.9.1" LIBJSON="UNKNOWN" RASTER + (1 row) + +#. Install some other packages needed by the extension dependencies:: + + sudo apt-get install python-dev libxml2-dev libxslt1-dev libgeos-c1 + + +Ubuntu 12.04 (PostgreSQL 9.1 and PostGIS 1.5) ++++++++++++++++++++++++++++++++++++++++++++++ #. Install PostGIS:: @@ -39,19 +76,9 @@ All commands assume an existing CKAN database named ``ckan_default``. #. Change the owner to spatial tables to the CKAN user to avoid errors later on:: - - Open the Postgres console:: - - $ sudo -u postgres psql - - Connect to the ``ckan_default`` database:: - - postgres=# \c ckan_default - - Change the ownership for two spatial tables:: - ALTER TABLE spatial_ref_sys OWNER TO ckan_default; - ALTER TABLE geometry_columns OWNER TO ckan_default; + sudo -u postgres psql -d ckan_default -c 'ALTER TABLE geometry_columns OWNER TO ckan_default;' + sudo -u postgres psql -d ckan_default -c 'ALTER TABLE spatial_ref_sys OWNER TO ckan_default;' #. Execute the following command to see if PostGIS was properly installed:: @@ -143,6 +170,48 @@ Troubleshooting Here are some common problems you may find when installing or using the extension: +When upgrading the extension to a newer version ++++++++++++++++++++++++++++++++++++++++++++++++ + +:: + + File "/home/adria/dev/pyenvs/spatial/src/ckanext-spatial/ckanext/spatial/plugin.py", line 39, in + check_geoalchemy_requirement() + File "/home/adria/dev/pyenvs/spatial/src/ckanext-spatial/ckanext/spatial/plugin.py", line 37, in check_geoalchemy_requirement + raise ImportError(msg.format('geoalchemy')) + ImportError: This version of ckanext-spatial requires geoalchemy2. Please install it by running `pip install geoalchemy2`. + For more details see the "Troubleshooting" section of the install documentation + +Starting from CKAN 2.3, the spatial requires GeoAlchemy2_ instead of GeoAlchemy, as this +is incompatible with the SQLAlchemy version that CKAN core uses. GeoAlchemy2 will get +installed on a new deployment, but if you are upgrading an existing ckanext-spatial +install you'll need to install it manually. With the virtualenv CKAN is installed on +activated, run:: + + pip install GeoAlchemy2 + +Restart the server for the changes to take effect. + +:: + + File "/home/adria/dev/pyenvs/spatial/src/ckanext-spatial/ckanext/spatial/plugin.py", line 30, in check_geoalchemy_requirement + import geoalchemy2 + File "/home/adria/dev/pyenvs/spatial/local/lib/python2.7/site-packages/geoalchemy2/__init__.py", line 1, in + from .types import ( # NOQA + File "/home/adria/dev/pyenvs/spatial/local/lib/python2.7/site-packages/geoalchemy2/types.py", line 15, in + from .comparator import BaseComparator, Comparator + File "/home/adria/dev/pyenvs/spatial/local/lib/python2.7/site-packages/geoalchemy2/comparator.py", line 52, in + class BaseComparator(UserDefinedType.Comparator): + AttributeError: type object 'UserDefinedType' has no attribute 'Comparator' + +You are trying to run the extension against CKAN 2.3, but the requirements for CKAN haven't been updated +(GeoAlchemy2 is crashing against SQLAlchemy 0.7.x). Upgrade the CKAN requirements as described in the +`upgrade documentation`_. + +.. _GeoAlchemy2: http://geoalchemy-2.readthedocs.org/en/0.2.4/ +.. _upgrade documentation: http://docs.ckan.org/en/latest/maintaining/upgrading/upgrade-source.html + + When initializing the spatial tables ++++++++++++++++++++++++++++++++++++ diff --git a/doc/spatial-search.rst b/doc/spatial-search.rst index 424cd91..416fe3b 100644 --- a/doc/spatial-search.rst +++ b/doc/spatial-search.rst @@ -42,14 +42,14 @@ Regardless of the backend that you are using, in order to make a dataset queryable by location, an special extra must be defined, with its key named 'spatial'. The value must be a valid GeoJSON_ geometry, for example:: - { + { "type":"Polygon", "coordinates":[[[2.05827, 49.8625],[2.05827, 55.7447], [-6.41736, 55.7447], [-6.41736, 49.8625], [2.05827, 49.8625]]] } or:: - { + { "type": "Point", "coordinates": [-3.145,53.078] } @@ -70,11 +70,11 @@ The following table summarizes the different spatial search backends: +------------------------+---------------+-------------------------------------+-----------------------------------------------------------+-------------------------------------------+ | Backend | Solr Versions | Supported geometries | Sorting and relevance | Performance with large number of datasets | +========================+===============+=====================================+===========================================================+===========================================+ -| ``solr`` | 3.1 to 4.x | Bounding Box | Yes, spatial sorting combined with other query parameters | Good | +| ``solr`` | >= 3.1 | Bounding Box | Yes, spatial sorting combined with other query parameters | Good | +------------------------+---------------+-------------------------------------+-----------------------------------------------------------+-------------------------------------------+ -| ``solr-spatial-field`` | 4.x | Bounding Box, Point and Polygon [1] | Not implemented | Good | +| ``solr-spatial-field`` | >= 4.x | Bounding Box, Point and Polygon [1] | Not implemented | Good | +------------------------+---------------+-------------------------------------+-----------------------------------------------------------+-------------------------------------------+ -| ``postgis`` | 1.3 to 4.x | Bounding Box | Partial, only spatial sorting supported [2] | Poor | +| ``postgis`` | >= 1.3 | Bounding Box | Partial, only spatial sorting supported [2] | Poor | +------------------------+---------------+-------------------------------------+-----------------------------------------------------------+-------------------------------------------+ @@ -196,9 +196,9 @@ There are snippets already created to load the map on the left sidebar or in the main body of the dataset details page, but these can be easily modified to suit your project needs -To add a map to the sidebar, add the following block to the dataset details page template (eg -``ckanext-myproj/ckanext/myproj/templates/package/read.html``). If your custom -theme is simply extending the CKAN default theme, you will need to add ``{% ckan_extends %}`` +To add a map to the sidebar, add the following block to the dataset page template (eg +``ckanext-myproj/ckanext/myproj/templates/package/read_base.html``). If your custom +theme is simply extending the CKAN default theme, you will need to add ``{% ckan_extends %}`` to the start of your custom read.html, then continue with this:: {% block secondary_content %} @@ -211,7 +211,8 @@ to the start of your custom read.html, then continue with this:: {% endblock %} -For adding the map to the main body, add this:: +For adding the map to the main body, add this to the main dataset page template (eg +``ckanext-myproj/ckanext/myproj/templates/package/read.html``):: {% block primary_content_inner %} diff --git a/pip-requirements.txt b/pip-requirements.txt index 2f9df16..84167e5 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -1,4 +1,5 @@ GeoAlchemy>=0.6 +GeoAlchemy2>=0.2.4 Shapely>=1.2.13 OWSLib==0.8.6 lxml>=2.3 diff --git a/setup.py b/setup.py index 1822fb1..484edda 100644 --- a/setup.py +++ b/setup.py @@ -48,5 +48,9 @@ setup( spatial=ckanext.spatial.commands.spatial:Spatial ckan-pycsw=ckanext.spatial.commands.csw:Pycsw validation=ckanext.spatial.commands.validation:Validation + + [ckan.test_plugins] + test_spatial_plugin = ckanext.spatial.tests.test_plugin.plugin:TestSpatialPlugin + """, ) diff --git a/test-core.ini b/test-core.ini deleted file mode 100644 index 5f80bb6..0000000 --- a/test-core.ini +++ /dev/null @@ -1,58 +0,0 @@ -[DEFAULT] -debug = true -# Uncomment and replace with the address which should receive any error reports -#email_to = you@yourdomain.com -smtp_server = localhost -error_email_from = paste@localhost - -[server:main] -use = egg:Paste#http -host = 0.0.0.0 -port = 5000 - - -[app:main] -use = config:../ckan/test-core.ini -# Here we hard-code the database and a flag to make default tests -# run fast. -ckan.plugins = harvest spatial_metadata spatial_query spatial_query_widget dataset_extent_map wms_preview spatial_harvest_metadata_api synchronous_search gemini_csw_harvester gemini_doc_harvester gemini_waf_harvester cswserver -ckan.spatial.srid = 4326 -ckan.spatial.default_map_extent=-6.88,49.74,0.50,59.2 -ckan.spatial.testing = true -ckan.spatial.validator.profiles = iso19139,constraints,gemini2 -# NB: other test configuration should go in test-core.ini, which is -# what the postgres tests use. - - -# Logging configuration -[loggers] -keys = root, ckan, sqlalchemy - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console - -[logger_ckan] -qualname = ckan -handlers = -level = INFO - -[logger_sqlalchemy] -handlers = -qualname = sqlalchemy.engine -level = WARN - -[handler_console] -class = StreamHandler -args = (sys.stdout,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/test.ini b/test.ini index abd6a00..84bd7cf 100644 --- a/test.ini +++ b/test.ini @@ -1,5 +1,5 @@ [DEFAULT] -debug = true +debug = false # Uncomment and replace with the address which should receive any error reports #email_to = you@yourdomain.com smtp_server = localhost @@ -12,18 +12,20 @@ port = 5000 [app:main] -use = config:test-core.ini -# Here we hard-code the database and a flag to make default tests -# run fast. -faster_db_test_hacks = True -sqlalchemy.url = sqlite:/// +use = config:../ckan/test-core.ini +ckan.legacy_templates = false +ckan.plugins = test_spatial_plugin harvest spatial_metadata spatial_query spatial_harvest_metadata_api gemini_csw_harvester gemini_doc_harvester gemini_waf_harvester cswserver +ckan.spatial.srid = 4326 +ckan.spatial.default_map_extent=-6.88,49.74,0.50,59.2 +ckan.spatial.testing = true +ckan.spatial.validator.profiles = iso19139,constraints,gemini2 # NB: other test configuration should go in test-core.ini, which is # what the postgres tests use. # Logging configuration [loggers] -keys = root, ckan, ckanext, sqlalchemy +keys = root, ckan, sqlalchemy [handlers] keys = console @@ -37,18 +39,11 @@ handlers = console [logger_ckan] qualname = ckan -handlers = console +handlers = level = INFO -propagate = 0 - -[logger_ckanext] -qualname = ckanext -handlers = console -level = DEBUG -propagate = 0 [logger_sqlalchemy] -handlers = console +handlers = qualname = sqlalchemy.engine level = WARN