From 0e8a62fe1e55b6b3f6855e346621b12446c3d504 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 19 Oct 2012 14:23:34 +0100 Subject: [PATCH] Reorganise XML test files into more sensible directory names. Add lower level validation tests. --- ckanext/spatial/tests/test_harvest.py | 56 +- ckanext/spatial/tests/test_validation.py | 139 +++++ .../{waf => xml/gemini2.1-waf}/index.html | 0 .../{waf => xml/gemini2.1-waf}/wales1.xml | 0 .../{waf => xml/gemini2.1-waf}/wales2.xml | 0 .../{single => xml/gemini2.1}/dataset1.xml | 0 .../gemini2.1}/error_bad_xml.xml | 0 .../gemini2.1}/error_validation.xml | 0 .../{single => xml/gemini2.1}/service1.xml | 0 .../gemini2.1}/service1_newer.xml | 0 .../gemini2.1}/source1/same_dataset.xml | 0 .../gemini2.1}/source2/same_dataset.xml | 0 ...01_Dataset_Invalid_XSD_No_Such_Element.xml | 0 ...taset_Invalid_XSD_No_Such_Element_unix.xml | 0 ...aset_Invalid_19139_Missing_Data_Format.xml | 0 ...Dataset_Invalid_GEMINI_Missing_Keyword.xml | 0 .../validation/04_Dataset_Valid.xml | 0 .../05_Series_Invalid_XSD_No_Such_Element.xml | 0 ...ries_Invalid_19139_Missing_Data_Format.xml | 0 ..._Series_Invalid_GEMINI_Missing_Keyword.xml | 0 .../gemini2.1}/validation/08_Series_Valid.xml | 0 .../09_Service_Invalid_No_Such_Element.xml | 0 ...ervice_Invalid_19139_Level_Description.xml | 0 ...11_Service_Invalid_GEMINI_Service_Type.xml | 0 .../validation/12_Service_Valid.xml | 0 .../13_Dataset_Invalid_Element_srv.xml | 0 .../tests/xml/iso19139/dataset-invalid.xml | 498 ++++++++++++++++++ .../spatial/tests/xml/iso19139/dataset.xml | 495 +++++++++++++++++ ckanext/spatial/tests/xml_file_server.py | 5 +- ckanext/spatial/validation/validation.py | 37 +- test.ini | 62 +++ 31 files changed, 1227 insertions(+), 65 deletions(-) create mode 100644 ckanext/spatial/tests/test_validation.py rename ckanext/spatial/tests/{waf => xml/gemini2.1-waf}/index.html (100%) rename ckanext/spatial/tests/{waf => xml/gemini2.1-waf}/wales1.xml (100%) rename ckanext/spatial/tests/{waf => xml/gemini2.1-waf}/wales2.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/dataset1.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/error_bad_xml.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/error_validation.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/service1.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/service1_newer.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/source1/same_dataset.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/source2/same_dataset.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/01_Dataset_Invalid_XSD_No_Such_Element.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/01_Dataset_Invalid_XSD_No_Such_Element_unix.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/02_Dataset_Invalid_19139_Missing_Data_Format.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/03_Dataset_Invalid_GEMINI_Missing_Keyword.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/04_Dataset_Valid.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/05_Series_Invalid_XSD_No_Such_Element.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/06_Series_Invalid_19139_Missing_Data_Format.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/07_Series_Invalid_GEMINI_Missing_Keyword.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/08_Series_Valid.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/09_Service_Invalid_No_Such_Element.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/10_Service_Invalid_19139_Level_Description.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/11_Service_Invalid_GEMINI_Service_Type.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/12_Service_Valid.xml (100%) rename ckanext/spatial/tests/{single => xml/gemini2.1}/validation/13_Dataset_Invalid_Element_srv.xml (100%) create mode 100644 ckanext/spatial/tests/xml/iso19139/dataset-invalid.xml create mode 100644 ckanext/spatial/tests/xml/iso19139/dataset.xml create mode 100644 test.ini diff --git a/ckanext/spatial/tests/test_harvest.py b/ckanext/spatial/tests/test_harvest.py index 2107133..f279161 100644 --- a/ckanext/spatial/tests/test_harvest.py +++ b/ckanext/spatial/tests/test_harvest.py @@ -134,14 +134,14 @@ class TestHarvest(HarvestFixtureBase): @classmethod def setup_class(cls): - SpatialHarvester._validator = Validators(profiles=['iso19139','gemini2']) + SpatialHarvester._validator = Validators(profiles=['gemini2']) HarvestFixtureBase.setup_class() def test_harvest_basic(self): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/waf/index.html', + 'url': u'http://127.0.0.1:8999/gemini2.1-waf/index.html', 'type': u'gemini-waf' } @@ -178,7 +178,7 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/service1.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/service1.xml', 'type': u'gemini-single' } @@ -285,7 +285,7 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/dataset1.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml', 'type': u'gemini-single' } @@ -386,7 +386,7 @@ class TestHarvest(HarvestFixtureBase): def test_harvest_error_bad_xml(self): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/error_bad_xml.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/error_bad_xml.xml', 'type': u'gemini-single' } @@ -410,7 +410,7 @@ class TestHarvest(HarvestFixtureBase): def test_harvest_error_404(self): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/not_there.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/not_there.xml', 'type': u'gemini-single' } @@ -430,7 +430,7 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/error_validation.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/error_validation.xml', 'type': u'gemini-single' } @@ -473,7 +473,7 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/dataset1.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml', 'type': u'gemini-single' } @@ -537,7 +537,7 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/service1.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/service1.xml', 'type': u'gemini-single' } @@ -575,7 +575,7 @@ class TestHarvest(HarvestFixtureBase): # Harvest an updated document, with a more recent modified date, package should be # updated and reactivated - source.url = u'http://127.0.0.1:8999/single/service1_newer.xml' + source.url = u'http://127.0.0.1:8999/gemini2.1/service1_newer.xml' source.save() third_job = self._create_job(source.id) @@ -608,7 +608,7 @@ class TestHarvest(HarvestFixtureBase): # Create source1 source1_fixture = { - 'url': u'http://127.0.0.1:8999/single/source1/same_dataset.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/source1/same_dataset.xml', 'type': u'gemini-single' } @@ -628,7 +628,7 @@ class TestHarvest(HarvestFixtureBase): # (As of https://github.com/okfn/ckanext-inspire/commit/9fb67 # we are no longer throwing an exception when this happens) source2_fixture = { - 'url': u'http://127.0.0.1:8999/single/source2/same_dataset.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/source2/same_dataset.xml', 'type': u'gemini-single' } @@ -673,7 +673,7 @@ class TestHarvest(HarvestFixtureBase): # Create source1 source1_fixture = { - 'url': u'http://127.0.0.1:8999/single/source1/same_dataset.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/source1/same_dataset.xml', 'type': u'gemini-single' } @@ -694,7 +694,7 @@ class TestHarvest(HarvestFixtureBase): # Harvest the same document, unchanged, from another source source2_fixture = { - 'url': u'http://127.0.0.1:8999/single/source2/same_dataset.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/source2/same_dataset.xml', 'type': u'gemini-single' } @@ -716,7 +716,7 @@ class TestHarvest(HarvestFixtureBase): # Create source1 source1_fixture = { - 'url': u'http://127.0.0.1:8999/single/service1.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/service1.xml', 'type': u'gemini-single' } @@ -733,7 +733,7 @@ class TestHarvest(HarvestFixtureBase): # Harvest the same document GUID but with a newer date, from another source. source2_fixture = { - 'url': u'http://127.0.0.1:8999/single/service1_newer.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/service1_newer.xml', 'type': u'gemini-single' } @@ -758,7 +758,7 @@ class TestHarvest(HarvestFixtureBase): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/dataset1.xml', + 'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml', 'type': u'gemini-single' } @@ -804,16 +804,6 @@ class TestHarvest(HarvestFixtureBase): assert len(source_dict['status']['packages']) == 1 -validation_error_xml = ''' - - - - Service type shall be one of 'discovery', 'view', 'download', 'transformation', 'invoke' or 'other' following INSPIRE generic names. - - - -''' - class TestValidation(HarvestFixtureBase): @classmethod @@ -824,7 +814,7 @@ class TestValidation(HarvestFixtureBase): def get_validation_errors(self, validation_test_filename): # Create source source_fixture = { - 'url': u'http://127.0.0.1:8999/single/validation/%s' % validation_test_filename, + 'url': u'http://127.0.0.1:8999/gemini2.1/validation/%s' % validation_test_filename, 'type': u'gemini-single' } @@ -839,16 +829,6 @@ class TestValidation(HarvestFixtureBase): errors = '; '.join([gather_error.message for gather_error in job.gather_errors]) return errors - def test_schematron_error_extraction(self): - failure_xml = lxml.etree.fromstring(validation_error_xml) - fail_element = failure_xml.getchildren()[0] - details = SchematronValidator.extract_error_details(fail_element) - if isinstance(details, tuple): - details = details[1] - assert_in("srv:serviceType/*[1] = 'discovery'", details) - assert_in("/*[local-name()='MD_Metadata'", details) - assert_in("Service type shall be one of 'discovery'", details) - def test_01_dataset_fail_iso19139_schema(self): errors = self.get_validation_errors('01_Dataset_Invalid_XSD_No_Such_Element.xml') assert len(errors) > 0 diff --git a/ckanext/spatial/tests/test_validation.py b/ckanext/spatial/tests/test_validation.py new file mode 100644 index 0000000..d437878 --- /dev/null +++ b/ckanext/spatial/tests/test_validation.py @@ -0,0 +1,139 @@ +import os + +from lxml import etree +from nose.tools import assert_equal, assert_in + +from ckanext.spatial import validation + +# other validation tests are in test_harvest.py + +class TestValidation: + + def get_validation_errors(self, validator, validation_test_filename): + validation_test_filepath = os.path.join(os.path.dirname(__file__), + 'xml', + validation_test_filename) + xml = etree.parse(validation_test_filepath) + is_valid, errors = validator.is_valid(xml) + return ';'.join(errors) + + def test_iso19139_failure(self): + errors = self.get_validation_errors(validation.ISO19139Schema, + 'iso19139/dataset-invalid.xml') + + assert len(errors) > 0 + assert_in('Dataset schema (gmx.xsd)', errors) + assert_in('{http://www.isotc211.org/2005/gmd}nosuchelement\': This element is not expected.', errors) + + def test_iso19139_pass(self): + errors = self.get_validation_errors(validation.ISO19139Schema, + 'iso19139/dataset.xml') + assert_equal(errors, '') + + # Gemini2.1 tests are basically the same as those in test_harvest.py but + # a few little differences make it worth not removing them in + # test_harvest + + def test_01_dataset_fail_iso19139_schema(self): + errors = self.get_validation_errors(validation.ISO19139EdenSchema, + 'gemini2.1/validation/01_Dataset_Invalid_XSD_No_Such_Element.xml') + assert len(errors) > 0 + assert_in('(gmx.xsd)', errors) + assert_in('\'{http://www.isotc211.org/2005/gmd}nosuchelement\': This element is not expected.', errors) + + def test_02_dataset_fail_constraints_schematron(self): + errors = self.get_validation_errors(validation.ConstraintsSchematron, + 'gemini2.1/validation/02_Dataset_Invalid_19139_Missing_Data_Format.xml') + assert len(errors) > 0 + assert_in('MD_Distribution / MD_Format: count(distributionFormat + distributorFormat) > 0', errors) + + def test_03_dataset_fail_gemini_schematron(self): + errors = self.get_validation_errors(validation.Gemini2Schematron, + 'gemini2.1/validation/03_Dataset_Invalid_GEMINI_Missing_Keyword.xml') + assert len(errors) > 0 + assert_in('Descriptive keywords are mandatory', errors) + + def assert_passes_all_gemini2_1_validation(self, xml_filepath): + errs = self.get_validation_errors(validation.ISO19139EdenSchema, + xml_filepath) + assert not errs, 'ISO19139EdenSchema: ' + errs + errs = self.get_validation_errors(validation.ConstraintsSchematron, + xml_filepath) + assert not errs, 'ConstraintsSchematron: ' + errs + errs = self.get_validation_errors(validation.Gemini2Schematron, + xml_filepath) + assert not errs, 'Gemini2Schematron: ' + errs + + def test_04_dataset_valid(self): + self.assert_passes_all_gemini2_1_validation('gemini2.1/validation/04_Dataset_Valid.xml') + + def test_05_series_fail_iso19139_schema(self): + errors = self.get_validation_errors(validation.ISO19139EdenSchema, + 'gemini2.1/validation/05_Series_Invalid_XSD_No_Such_Element.xml') + assert len(errors) > 0 + assert_in('(gmx.xsd)', errors) + assert_in('\'{http://www.isotc211.org/2005/gmd}nosuchelement\': This element is not expected.', errors) + + def test_06_series_fail_constraints_schematron(self): + errors = self.get_validation_errors(validation.ConstraintsSchematron, + 'gemini2.1/validation/06_Series_Invalid_19139_Missing_Data_Format.xml') + assert len(errors) > 0 + assert_in('MD_Distribution / MD_Format: count(distributionFormat + distributorFormat) > 0', errors) + + def test_07_series_fail_gemini_schematron(self): + errors = self.get_validation_errors(validation.Gemini2Schematron, + 'gemini2.1/validation/07_Series_Invalid_GEMINI_Missing_Keyword.xml') + assert len(errors) > 0 + assert_in('Descriptive keywords are mandatory', errors) + + def test_08_series_valid(self): + self.assert_passes_all_gemini2_1_validation('gemini2.1/validation/08_Series_Valid.xml') + + def test_09_service_fail_iso19139_schema(self): + errors = self.get_validation_errors(validation.ISO19139EdenSchema, + 'gemini2.1/validation/09_Service_Invalid_No_Such_Element.xml') + assert len(errors) > 0 + assert_in('(gmx.xsd & srv.xsd)', errors) + assert_in('\'{http://www.isotc211.org/2005/gmd}nosuchelement\': This element is not expected.', errors) + + def test_10_service_fail_constraints_schematron(self): + errors = self.get_validation_errors(validation.ConstraintsSchematron, + 'gemini2.1/validation/10_Service_Invalid_19139_Level_Description.xml') + assert len(errors) > 0 + assert_in("DQ_Scope: 'levelDescription' is mandatory if 'level' notEqual 'dataset' or 'series'.", errors) + + def test_11_service_fail_gemini_schematron(self): + errors = self.get_validation_errors(validation.Gemini2Schematron, + 'gemini2.1/validation/11_Service_Invalid_GEMINI_Service_Type.xml') + assert len(errors) > 0 + assert_in("Service type shall be one of 'discovery', 'view', 'download', 'transformation', 'invoke' or 'other' following INSPIRE generic names.", errors) + + def test_12_service_valid(self): + self.assert_passes_all_gemini2_1_validation('gemini2.1/validation/12_Service_Valid.xml') + + def test_13_dataset_fail_iso19139_schema_2(self): + # This test Dataset has srv tags and only Service metadata should. + errors = self.get_validation_errors(validation.ISO19139EdenSchema, + 'gemini2.1/validation/13_Dataset_Invalid_Element_srv.xml') + assert len(errors) > 0 + assert_in('(gmx.xsd)', errors) + assert_in('(u"Element \'{http://www.isotc211.org/2005/srv}SV_ServiceIdentification\': This element is not expected.', errors) + + def test_schematron_error_extraction(self): + validation_error_xml = ''' + + + + Service type shall be one of 'discovery', 'view', 'download', 'transformation', 'invoke' or 'other' following INSPIRE generic names. + + + +''' + failure_xml = etree.fromstring(validation_error_xml) + fail_element = failure_xml.getchildren()[0] + details = validation.SchematronValidator.extract_error_details(fail_element) + if isinstance(details, tuple): + details = details[1] + assert_in("srv:serviceType/*[1] = 'discovery'", details) + assert_in("/*[local-name()='MD_Metadata'", details) + assert_in("Service type shall be one of 'discovery'", details) diff --git a/ckanext/spatial/tests/waf/index.html b/ckanext/spatial/tests/xml/gemini2.1-waf/index.html similarity index 100% rename from ckanext/spatial/tests/waf/index.html rename to ckanext/spatial/tests/xml/gemini2.1-waf/index.html diff --git a/ckanext/spatial/tests/waf/wales1.xml b/ckanext/spatial/tests/xml/gemini2.1-waf/wales1.xml similarity index 100% rename from ckanext/spatial/tests/waf/wales1.xml rename to ckanext/spatial/tests/xml/gemini2.1-waf/wales1.xml diff --git a/ckanext/spatial/tests/waf/wales2.xml b/ckanext/spatial/tests/xml/gemini2.1-waf/wales2.xml similarity index 100% rename from ckanext/spatial/tests/waf/wales2.xml rename to ckanext/spatial/tests/xml/gemini2.1-waf/wales2.xml diff --git a/ckanext/spatial/tests/single/dataset1.xml b/ckanext/spatial/tests/xml/gemini2.1/dataset1.xml similarity index 100% rename from ckanext/spatial/tests/single/dataset1.xml rename to ckanext/spatial/tests/xml/gemini2.1/dataset1.xml diff --git a/ckanext/spatial/tests/single/error_bad_xml.xml b/ckanext/spatial/tests/xml/gemini2.1/error_bad_xml.xml similarity index 100% rename from ckanext/spatial/tests/single/error_bad_xml.xml rename to ckanext/spatial/tests/xml/gemini2.1/error_bad_xml.xml diff --git a/ckanext/spatial/tests/single/error_validation.xml b/ckanext/spatial/tests/xml/gemini2.1/error_validation.xml similarity index 100% rename from ckanext/spatial/tests/single/error_validation.xml rename to ckanext/spatial/tests/xml/gemini2.1/error_validation.xml diff --git a/ckanext/spatial/tests/single/service1.xml b/ckanext/spatial/tests/xml/gemini2.1/service1.xml similarity index 100% rename from ckanext/spatial/tests/single/service1.xml rename to ckanext/spatial/tests/xml/gemini2.1/service1.xml diff --git a/ckanext/spatial/tests/single/service1_newer.xml b/ckanext/spatial/tests/xml/gemini2.1/service1_newer.xml similarity index 100% rename from ckanext/spatial/tests/single/service1_newer.xml rename to ckanext/spatial/tests/xml/gemini2.1/service1_newer.xml diff --git a/ckanext/spatial/tests/single/source1/same_dataset.xml b/ckanext/spatial/tests/xml/gemini2.1/source1/same_dataset.xml similarity index 100% rename from ckanext/spatial/tests/single/source1/same_dataset.xml rename to ckanext/spatial/tests/xml/gemini2.1/source1/same_dataset.xml diff --git a/ckanext/spatial/tests/single/source2/same_dataset.xml b/ckanext/spatial/tests/xml/gemini2.1/source2/same_dataset.xml similarity index 100% rename from ckanext/spatial/tests/single/source2/same_dataset.xml rename to ckanext/spatial/tests/xml/gemini2.1/source2/same_dataset.xml diff --git a/ckanext/spatial/tests/single/validation/01_Dataset_Invalid_XSD_No_Such_Element.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/01_Dataset_Invalid_XSD_No_Such_Element.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/01_Dataset_Invalid_XSD_No_Such_Element.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/01_Dataset_Invalid_XSD_No_Such_Element.xml diff --git a/ckanext/spatial/tests/single/validation/01_Dataset_Invalid_XSD_No_Such_Element_unix.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/01_Dataset_Invalid_XSD_No_Such_Element_unix.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/01_Dataset_Invalid_XSD_No_Such_Element_unix.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/01_Dataset_Invalid_XSD_No_Such_Element_unix.xml diff --git a/ckanext/spatial/tests/single/validation/02_Dataset_Invalid_19139_Missing_Data_Format.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/02_Dataset_Invalid_19139_Missing_Data_Format.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/02_Dataset_Invalid_19139_Missing_Data_Format.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/02_Dataset_Invalid_19139_Missing_Data_Format.xml diff --git a/ckanext/spatial/tests/single/validation/03_Dataset_Invalid_GEMINI_Missing_Keyword.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/03_Dataset_Invalid_GEMINI_Missing_Keyword.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/03_Dataset_Invalid_GEMINI_Missing_Keyword.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/03_Dataset_Invalid_GEMINI_Missing_Keyword.xml diff --git a/ckanext/spatial/tests/single/validation/04_Dataset_Valid.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/04_Dataset_Valid.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/04_Dataset_Valid.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/04_Dataset_Valid.xml diff --git a/ckanext/spatial/tests/single/validation/05_Series_Invalid_XSD_No_Such_Element.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/05_Series_Invalid_XSD_No_Such_Element.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/05_Series_Invalid_XSD_No_Such_Element.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/05_Series_Invalid_XSD_No_Such_Element.xml diff --git a/ckanext/spatial/tests/single/validation/06_Series_Invalid_19139_Missing_Data_Format.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/06_Series_Invalid_19139_Missing_Data_Format.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/06_Series_Invalid_19139_Missing_Data_Format.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/06_Series_Invalid_19139_Missing_Data_Format.xml diff --git a/ckanext/spatial/tests/single/validation/07_Series_Invalid_GEMINI_Missing_Keyword.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/07_Series_Invalid_GEMINI_Missing_Keyword.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/07_Series_Invalid_GEMINI_Missing_Keyword.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/07_Series_Invalid_GEMINI_Missing_Keyword.xml diff --git a/ckanext/spatial/tests/single/validation/08_Series_Valid.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/08_Series_Valid.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/08_Series_Valid.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/08_Series_Valid.xml diff --git a/ckanext/spatial/tests/single/validation/09_Service_Invalid_No_Such_Element.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/09_Service_Invalid_No_Such_Element.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/09_Service_Invalid_No_Such_Element.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/09_Service_Invalid_No_Such_Element.xml diff --git a/ckanext/spatial/tests/single/validation/10_Service_Invalid_19139_Level_Description.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/10_Service_Invalid_19139_Level_Description.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/10_Service_Invalid_19139_Level_Description.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/10_Service_Invalid_19139_Level_Description.xml diff --git a/ckanext/spatial/tests/single/validation/11_Service_Invalid_GEMINI_Service_Type.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/11_Service_Invalid_GEMINI_Service_Type.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/11_Service_Invalid_GEMINI_Service_Type.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/11_Service_Invalid_GEMINI_Service_Type.xml diff --git a/ckanext/spatial/tests/single/validation/12_Service_Valid.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/12_Service_Valid.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/12_Service_Valid.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/12_Service_Valid.xml diff --git a/ckanext/spatial/tests/single/validation/13_Dataset_Invalid_Element_srv.xml b/ckanext/spatial/tests/xml/gemini2.1/validation/13_Dataset_Invalid_Element_srv.xml similarity index 100% rename from ckanext/spatial/tests/single/validation/13_Dataset_Invalid_Element_srv.xml rename to ckanext/spatial/tests/xml/gemini2.1/validation/13_Dataset_Invalid_Element_srv.xml diff --git a/ckanext/spatial/tests/xml/iso19139/dataset-invalid.xml b/ckanext/spatial/tests/xml/iso19139/dataset-invalid.xml new file mode 100644 index 0000000..a299e03 --- /dev/null +++ b/ckanext/spatial/tests/xml/iso19139/dataset-invalid.xml @@ -0,0 +1,498 @@ + + + + test-record + + + test-dataset + + + eng + + + dataset + + + + + Lachlan Renwick + + + Scottish Natural Heritage + + + Geographic Systems and Data Coordinator + + + + + + + 01463 725000 + + + + + + + Great Glen House, Leachkin Road + + + INVERNESS + + + IV3 8NW + + + United Kingdom + + + data_supply@snh.gov.uk + + + + + + + pointOfContact + + + + + 2011-09-23T10:06:08 + + + + + + + urn:ogc:def:crs:EPSG::27700 + + + + + + + + + + + Country Parks (Scotland) + + + + + 2004-02 + + + creation + + + + + + + 2006-07-03 + + + revision + + + + + + + CPK + + + + + + + + + + Parks are set up by Local Authorities to provide open-air recreation facilities close to towns and cities. [edited] + + + + + Lachlan Renwick + + + Scottish Natural Heritage + + + Geographic Systems & Data Coordinator + + + + + + + 01463 725000 + + + + + + + Great Glen House, Leachkin Road + + + INVERNESS + + + IV3 8NW + + + United Kingdom + + + data_supply@snh.gov.uk + + + + + + + custodian + + + + + + + Lachlan Renwick + + + Scottish Natural Heritage + + + Geographic Systems & Data Coordinator + + + + + + + 01463 725000 + + + + + + + Great Glen House, Leachkin Road + + + INVERNESS + + + IV3 8NW + + + United Kingdom + + + data_supply@snh.gov.uk + + + + + + + distributor + + + + + + + irregular + + + + + + + SDE Feature Class + + + + + + + + + + Nature conservation + + + + + Government Category List + + + + + 2004-07-15 + + + revision + + + + + + + + + + + copyright + + + otherRestrictions + + + copyright + + + otherRestrictions + + + Copyright Scottish Natural Heritage + + + + + + + Reference and PSMA Only + + + http://www.test.gov.uk/licenseurl + + + + + + + + + + 5 + + + + + eng + + + environment + + + + + + + + + + + ISO 3166 + + + + + 2007-09-02 + + + revision + + + + + + + GB-SCT + + + + + + + + + -8.97114288 + + + 0.205857204 + + + 54.529947158 + + + 61.06066944 + + + + + + + + 1998 + 2010 + + + + + + + + + + + + + + ESRI Shapefile + + + Unknown + + + + + + + KML + + + 2.1 + + + + + + + GML + + + 3.1.1 + + + + + + + + + Lachlan Renwick + + + Scottish Natural Heritage + + + Geographic Systems & Data Coordinator + + + + + + + 01463 725000 + + + + + + + Great Glen House, Leachkin Road + + + INVERNESS + + + IV3 8NW + + + United Kingdom + + + data_supply@snh.gov.uk + + + + + + + distributor + + + + + + + SDE Feature Class + + + + + + + + + + + + http://www.snh.org.uk/snhi + + + + + + + + + + + + + https://gateway.snh.gov.uk/pls/apex_ddtdb2/f?p=101 + + + Test Resource Name + + + Test Resource Description + + + download + + + + + + + + + + + + + dataset + + + + + + + Country Park is not a statutory designation. Countryside (Scotland) Act 1967 Section 48 gives local authorities power to assess and review the need for Country Parks in consultation with SNH. + + + + + + diff --git a/ckanext/spatial/tests/xml/iso19139/dataset.xml b/ckanext/spatial/tests/xml/iso19139/dataset.xml new file mode 100644 index 0000000..885a9b5 --- /dev/null +++ b/ckanext/spatial/tests/xml/iso19139/dataset.xml @@ -0,0 +1,495 @@ + + + + test-dataset-1 + + + eng + + + dataset + + + + + Lachlan Renwick + + + Scottish Natural Heritage + + + Geographic Systems and Data Coordinator + + + + + + + 01463 725000 + + + + + + + Great Glen House, Leachkin Road + + + INVERNESS + + + IV3 8NW + + + United Kingdom + + + data_supply@snh.gov.uk + + + + + + + pointOfContact + + + + + 2011-09-23T10:06:08 + + + + + + + urn:ogc:def:crs:EPSG::27700 + + + + + + + + + + + Country Parks (Scotland) + + + + + 2004-02 + + + creation + + + + + + + 2006-07-03 + + + revision + + + + + + + CPK + + + + + + + + + + Parks are set up by Local Authorities to provide open-air recreation facilities close to towns and cities. [edited] + + + + + Lachlan Renwick + + + Scottish Natural Heritage + + + Geographic Systems & Data Coordinator + + + + + + + 01463 725000 + + + + + + + Great Glen House, Leachkin Road + + + INVERNESS + + + IV3 8NW + + + United Kingdom + + + data_supply@snh.gov.uk + + + + + + + custodian + + + + + + + Lachlan Renwick + + + Scottish Natural Heritage + + + Geographic Systems & Data Coordinator + + + + + + + 01463 725000 + + + + + + + Great Glen House, Leachkin Road + + + INVERNESS + + + IV3 8NW + + + United Kingdom + + + data_supply@snh.gov.uk + + + + + + + distributor + + + + + + + irregular + + + + + + + SDE Feature Class + + + + + + + + + + Nature conservation + + + + + Government Category List + + + + + 2004-07-15 + + + revision + + + + + + + + + + + copyright + + + otherRestrictions + + + copyright + + + otherRestrictions + + + Copyright Scottish Natural Heritage + + + + + + + Reference and PSMA Only + + + http://www.test.gov.uk/licenseurl + + + + + + + + + + 5 + + + + + eng + + + environment + + + + + + + + + + + ISO 3166 + + + + + 2007-09-02 + + + revision + + + + + + + GB-SCT + + + + + + + + + -8.97114288 + + + 0.205857204 + + + 54.529947158 + + + 61.06066944 + + + + + + + + 1998 + 2010 + + + + + + + + + + + + + + ESRI Shapefile + + + Unknown + + + + + + + KML + + + 2.1 + + + + + + + GML + + + 3.1.1 + + + + + + + + + Lachlan Renwick + + + Scottish Natural Heritage + + + Geographic Systems & Data Coordinator + + + + + + + 01463 725000 + + + + + + + Great Glen House, Leachkin Road + + + INVERNESS + + + IV3 8NW + + + United Kingdom + + + data_supply@snh.gov.uk + + + + + + + distributor + + + + + + + SDE Feature Class + + + + + + + + + + + + http://www.snh.org.uk/snhi + + + + + + + + + + + + + https://gateway.snh.gov.uk/pls/apex_ddtdb2/f?p=101 + + + Test Resource Name + + + Test Resource Description + + + download + + + + + + + + + + + + + dataset + + + + + + + Country Park is not a statutory designation. Countryside (Scotland) Act 1967 Section 48 gives local authorities power to assess and review the need for Country Parks in consultation with SNH. + + + + + + diff --git a/ckanext/spatial/tests/xml_file_server.py b/ckanext/spatial/tests/xml_file_server.py index 2aab64c..31e62f0 100644 --- a/ckanext/spatial/tests/xml_file_server.py +++ b/ckanext/spatial/tests/xml_file_server.py @@ -10,8 +10,9 @@ PORT = 8999 def serve(port=PORT): '''Serves test XML files over HTTP''' - # Make sure we serve from the tests directory - os.chdir(os.path.dirname(os.path.abspath( __file__ ))) + # Make sure we serve from the tests' XML directory + os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'xml')) Handler = SimpleHTTPServer.SimpleHTTPRequestHandler diff --git a/ckanext/spatial/validation/validation.py b/ckanext/spatial/validation/validation.py index 67ecb7e..c7e95d7 100644 --- a/ckanext/spatial/validation/validation.py +++ b/ckanext/spatial/validation/validation.py @@ -53,6 +53,18 @@ class XsdValidator(BaseValidator): return True, [] +class ISO19139Schema(XsdValidator): + name = 'iso19139' + title = 'ISO19139 XSD Schema' + + @classmethod + def is_valid(cls, xml): + xsd_path = 'xml/iso19139' + gmx_xsd_filepath = os.path.join(os.path.dirname(__file__), + xsd_path, 'gmx/gmx.xsd') + is_valid, errors = cls._is_valid(xml, gmx_xsd_filepath, 'Dataset schema (gmx.xsd)') + return is_valid, errors + class ISO19139EdenSchema(XsdValidator): name = 'iso19139eden' title = 'ISO19139 XSD Schema (EDEN)' @@ -157,31 +169,6 @@ class SchematronValidator(BaseValidator): return etree.XSLT(compiled) -class ISO19139Schema(SchematronValidator): - name = 'iso19139' - title = 'ISO19139 XSD Schema' - - @classmethod - def get_schematrons(cls): - with resource_stream("ckanext.csw", "xml/schematron/ExtractSchFromXSD.xsl") as xsl_file: - xsl = etree.parse(xsl_file) - xsd2sch = etree.XSLT(xsl) - - root = resource_filename("ckanext.csw", "xml/iso19139") - schematrons = [] - for filename in ["gmd/gmd.xsd"]: - filename = os.path.join(root, filename) - with open(filename) as xsd_file: - xsd = etree.parse(xsd_file) - extracted_schematron_rules = xsd2sch(xsd) - # There are no schematron rules here! So this validation is pointless. - # - # - # - # - schematrons.append(cls.schematron(extracted_schematron_rules)) - return schematrons - class ConstraintsSchematron(SchematronValidator): name = 'constraints' title = 'ISO19139 Table A.1 Constraints Schematron 1.3' diff --git a/test.ini b/test.ini new file mode 100644 index 0000000..abd6a00 --- /dev/null +++ b/test.ini @@ -0,0 +1,62 @@ +[DEFAULT] +debug = true +# Uncomment and replace with the address which should receive any error reports +#email_to = you@yourdomain.com +smtp_server = localhost +error_email_from = paste@localhost + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + + +[app:main] +use = config:test-core.ini +# Here we hard-code the database and a flag to make default tests +# run fast. +faster_db_test_hacks = True +sqlalchemy.url = sqlite:/// +# NB: other test configuration should go in test-core.ini, which is +# what the postgres tests use. + + +# Logging configuration +[loggers] +keys = root, ckan, ckanext, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_ckan] +qualname = ckan +handlers = console +level = INFO +propagate = 0 + +[logger_ckanext] +qualname = ckanext +handlers = console +level = DEBUG +propagate = 0 + +[logger_sqlalchemy] +handlers = console +qualname = sqlalchemy.engine +level = WARN + +[handler_console] +class = StreamHandler +args = (sys.stdout,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s