From 41896d312be82e26952b237bcfd6ca74c45c59f7 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Mon, 21 Jun 2021 15:20:39 +0200 Subject: [PATCH] fix for from and until values on temporal and native period fields --- .../ariadneplus/elasticsearch/BulkUpload.java | 28 +++++------ .../src/main/resources/application.properties | 4 +- .../read_collection_data_template.sparql | 21 ++++---- .../sparql/read_record_data_template.sparql | 8 ++-- .../GraphDbReaderAndESIndexTest.java | 48 +++++++++++-------- 5 files changed, 57 insertions(+), 52 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 9aacfd7..4c2bb1d 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -1,15 +1,12 @@ package eu.dnetlib.ariadneplus.elasticsearch; -import eu.dnetlib.ariadneplus.elasticsearch.model.*; +import eu.dnetlib.ariadneplus.elasticsearch.model.AriadnePlusEntry; +import eu.dnetlib.ariadneplus.elasticsearch.model.Spatial; import eu.dnetlib.ariadneplus.reader.ResourceManager; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpHost; -import org.apache.lucene.spatial3d.geom.GeoPoint; -import org.apache.lucene.spatial3d.geom.GeoPolygon; -import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; -import org.apache.lucene.spatial3d.geom.PlanetModel; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; @@ -17,18 +14,17 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.geo.builders.CoordinatesBuilder; -import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.geometry.Polygon; -import org.locationtech.jts.geom.Coordinate; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import java.io.IOException; -import java.io.OutputStream; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; @Service @@ -70,9 +66,9 @@ public class BulkUpload { AriadnePlusEntry ace = ((AriadnePlusEntry) next); if (isCollection) { ace.setResourceType("collection"); - if (ace.getSpatial()==null) { - ace.setSpatial(Arrays.asList(new Spatial())); - } +// if (ace.getSpatial()==null) { +// ace.setSpatial(Arrays.asList(new Spatial())); +// } } else { ace.setResourceType("dataset"); @@ -162,9 +158,9 @@ public class BulkUpload { // ace.getSpatial().removeIf(s -> (s.getPlaceName()!=null&&s.getPlaceName().equals("Name not provided")&&Objects.isNull(s.getLocation()))); // } } - else { - ace.setSpatial(Arrays.asList(new Spatial())); - } +// else { +// ace.setSpatial(Arrays.asList(new Spatial())); +// } } // if (ace.getTemporal()!=null) { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 7d44bd1..febb6f7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -61,7 +61,7 @@ class.map.specifications={\ "NativePeriod": {\ "class_type": "prototype",\ "mappings": {\ -"https://www.ariadne-infrastructure.eu/property/from": {\ +"https://www.ariadne-infrastructure.eu/property/nativeFrom": {\ "class_field": "From",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -71,7 +71,7 @@ class.map.specifications={\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"https://www.ariadne-infrastructure.eu/property/until": {\ +"https://www.ariadne-infrastructure.eu/property/nativeUntil": {\ "class_field": "Until",\ "substring": "no",\ "element_type": "java.lang.String"\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql index d09e64a..0597db8 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -49,8 +49,8 @@ CONSTRUCT { ?temporal aoprop:uri ?temporal . %record aoprop:temporalNative ?temporalNative . ?temporalNative aoprop:nativePeriodName ?temporalNativePeriodName . -?temporalNative aoprop:from ?temporalNativeFrom . -?temporalNative aoprop:until ?temporalNativeUntil . +?temporalNative aoprop:nativeFrom ?temporalNativeFrom . +?temporalNative aoprop:nativeUntil ?temporalNativeUntil . %record aoprop:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . %record aoprop:resourceType ?resourceType . @@ -113,19 +113,22 @@ where { graph { %record aocat:has_temporal_coverage ?temporalNative . ?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName . - } - graph { optional { ?temporalNative aocat:from ?temporalNativeFrom . ?temporalNative aocat:until ?temporalNativeUntil . } } } - graph { - optional { - %record aocat:has_temporal_coverage ?temporal . + } + } + + optional { + graph { + %record aocat:has_temporal_coverage ?temporal . + } + graph { ?temporal aocat:has_period ?label . - graph { + graph { ?label skos:prefLabel ?temporalPeriodName . } optional { @@ -134,8 +137,6 @@ where { } } } - } - } graph { { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index aec0535..e2de085 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -53,8 +53,8 @@ CONSTRUCT { ?temporal aoprop:uri ?temporal . %record aoprop:temporalNative ?temporalNative . ?temporalNative aoprop:nativePeriodName ?temporalNativePeriodName . -?temporalNative aoprop:from ?temporalNativeFrom . -?temporalNative aoprop:until ?temporalNativeUntil . +?temporalNative aoprop:nativeFrom ?temporalNativeFrom . +?temporalNative aoprop:nativeUntil ?temporalNativeUntil . %record aoprop:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . %record aoprop:resourceType ?resourceType . @@ -100,9 +100,7 @@ where { %record aocat:has_temporal_coverage ?temporalNative . ?temporalNative aocat:has_native_period ?nativeLabel . ?nativeLabel skos:prefLabel ?temporalNativePeriodName . - } - graph { - optional { + optional { ?temporalNative aocat:from ?temporalNativeFrom . ?temporalNative aocat:until ?temporalNativeUntil . } diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 4d1e98f..1fdc621 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -32,7 +32,7 @@ public class GraphDbReaderAndESIndexTest { private RunSPARQLQueryService runSPQRLQuery; @Test - @Ignore +// @Ignore public void uploadAMCRFieldworkTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/E61E0F4E-268F-39E4-8EDB-A431AFC505AA"; @@ -42,7 +42,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadAMCRDocumentTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/AMCR/FC59581D-DC3A-31DA-922A-98DE764F3D76"; @@ -52,7 +52,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadAMCRSiteTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/3C7EC936-A7CA-3720-B3DC-413A25754FD4"; @@ -62,7 +62,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadAMCRIndividualFindTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/98D717C4-410F-35C6-8072-FABA7686B4A3"; @@ -81,7 +81,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadADSRecordWithNativeFromUntilTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/3C3C7A86-FF09-3431-95B1-B9A4AA8293AF"; @@ -91,7 +91,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadADSRecordWithoutNativeFromUntilTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DF5F27D3-C877-3F23-9EAA-3776362363AA"; @@ -101,7 +101,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadZbivaRecordSpatialTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/ZRC-SAZU-Zbiva/B34517C6-8D94-3A02-B461-08522F958479"; @@ -111,7 +111,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadHNMCollectionSpatialTest() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/5A7A4257-EE73-31F9-9F74-BADB371555F5"; @@ -121,10 +121,10 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadHNMCollectionTemporalTest() throws Exception { boolean isRecord = false; - String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/0F1AF07E-CE63-342C-8E95-11FF1C024BD5"; + String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/1AE50143-45C7-304F-8367-BCF3606CEF10"; String datasource = "hnm"; String collectionId = "hnmad"; readAndIndexTest(isRecord, recordId, datasource, collectionId); @@ -141,8 +141,8 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore - public void uploadDansNoSpatialTest() throws Exception { +// @Ignore + public void uploadDansSpatialTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/F100A0AD-6A7F-3976-B77F-FFAB4F5B55DD"; String datasource = "dans"; @@ -151,8 +151,8 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore - public void uploadDansFieldworkReportTest() throws Exception { +// @Ignore + public void uploadDansNativePeriodTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CA076E46-5CED-322C-B77E-3B90C11B968B"; String datasource = "dans"; @@ -161,7 +161,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - public void uploadDansPolygonTest() throws Exception { + public void uploadDansTemporalPolygonTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B"; String datasource = "dans"; @@ -170,7 +170,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadSndRockartTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/SHFA/FED23426-2C68-3BB3-9BBA-24F2077C9C6A"; @@ -180,7 +180,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadNIAMTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/NIAM-BAS%2FAKB/D4388BF5-AF3D-3F24-8C2E-2FEAD1255FB1"; @@ -190,7 +190,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadADS398Test() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/AAA81A6D-56F3-341C-BAF0-791C31BC7F73"; @@ -200,7 +200,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadROADTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FC70B370-C489-31C5-B1D4-339CFD28CF2B"; @@ -209,6 +209,16 @@ public class GraphDbReaderAndESIndexTest { readAndIndexTest(isRecord, recordId, datasource, collectionId); } + @Test +// @Ignore + public void uploadADS328NativeFromUntilTest() throws Exception { + boolean isRecord = true; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/7516E8FC-7C1E-38F7-8D62-8BF96B0D2559"; + String datasource = "ads"; + String collectionId = "328"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties();