From 1545f3900f54c9b5290d3a332c1db35312f0ccfc Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Mon, 31 May 2021 16:43:21 +0200 Subject: [PATCH 01/13] get specific error for elastic search record parsing error --- .../java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 8b7e66c..4143947 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -171,6 +171,7 @@ public class BulkUpload { // log.info("Indexing to ES completed with status: " + bulkResponse.status()); if (bulkResponse.hasFailures()) { log.error("FailureMessage: " + bulkResponse.buildFailureMessage()); + esResponseCode = -7; } } else { From 1083d4e7231b08237996f40c61685aa5e1876887 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Mon, 31 May 2021 16:44:34 +0200 Subject: [PATCH 02/13] set new elastic search index on staging --- .../src/main/resources/application.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index f290220..8ff8ffb 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -5,12 +5,12 @@ server.port=8281 graphdb.serverUrl=http://graphdb-test.ariadne.d4science.org:7200 graphdb.writer.user=writer -graphdb.writer.pwd=pwd +graphdb.writer.pwd=*** graphdb.repository=ariadneplus-ts01 graphdb.baseURI=https://ariadne-infrastructure.eu/ elasticsearch.hostname=elastic-test.ariadne.d4science.org -elasticsearch.indexname=catalog_test +elasticsearch.indexname=ariadneplus catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/rdfType'][0]['value']=='Record')] catalog.entry.collection.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/rdfType'][0]['value']=='Collection')] From 06f20f7a8d64fd2042470231759ebeaf9a8f4101 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Mon, 31 May 2021 16:59:56 +0200 Subject: [PATCH 03/13] shell command to create the new elastic search index with mapping on staging --- .../es/schemaAriadneplusElasticsearch.curl | 331 ++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 dnet-ariadneplus/src/main/resources/es/schemaAriadneplusElasticsearch.curl diff --git a/dnet-ariadneplus/src/main/resources/es/schemaAriadneplusElasticsearch.curl b/dnet-ariadneplus/src/main/resources/es/schemaAriadneplusElasticsearch.curl new file mode 100644 index 0000000..2a04585 --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/es/schemaAriadneplusElasticsearch.curl @@ -0,0 +1,331 @@ +curl -X PUT "http://elastic-test.ariadne.d4science.org:9200/ariadneplus?pretty" -H 'Content-Type: application/json' -d ' +{ + "mappings": { + "properties": { + "derivedSubject": { + "properties": { + "prefLabel": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "source": { + "type": "text" + }, + "id": { + "type": "text" + }, + "lang": { + "type": "text" + } + } + }, + "accessPolicy": { + "type": "text" + }, + "accessRights": { + "type": "text" + }, + "ariadneSubject": { + "properties": { + "prefLabel": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + } + } + }, + "contributor": { + "properties": { + "email": { + "type": "text" + }, + "homepage": { + "type": "text" + }, + "institution": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "agentIdentifier": { + "type": "text" + } + } + }, + "creator": { + "properties": { + "email": { + "type": "text" + }, + "homepage": { + "type": "text" + }, + "institution": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "agentIdentifier": { + "type": "text" + } + } + }, + "description": { + "properties": { + "text": { + "type": "text" + }, + "language": { + "type": "text" + } + } + }, + "extent": { + "type": "text" + }, + "identifier": { + "type": "keyword" + }, + "isPartOf": { + "type": "text" + }, + "issued": { + "type": "date", + "format": "year || date_optional_time" + }, + "landingPage": { + "type": "text" + }, + "language": { + "type": "text" + }, + "modified": { + "type": "date", + "format": "year || date_optional_time" + }, + "nativeSubject": { + "properties": { + "prefLabel": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "rdfAbout": { + "type": "text" + } + } + }, + "originalId": { + "type": "text" + }, + "owner": { + "properties": { + "email": { + "type": "text" + }, + "homepage": { + "type": "text" + }, + "institution": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "agentIdentifier": { + "type": "text" + } + } + }, + "publisher": { + "properties": { + "email": { + "type": "text" + }, + "homepage": { + "type": "text" + }, + "institution": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "agentIdentifier": { + "type": "text" + } + } + }, + "is_about": { + "properties": { + "label": { + "type": "text" + }, + "uri": { + "type": "text" + } + } + }, + "resourceType": { + "type": "text" + }, + "has_type": { + "properties": { + "label": { + "type": "text" + }, + "uri": { + "type": "text" + } + } + }, + "responsible": { + "properties": { + "email": { + "type": "text" + }, + "homepage": { + "type": "text" + }, + "institution": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "agentIdentifier": { + "type": "text" + } + } + }, + "spatial": { + "type": "nested", + "properties": { + "placeName": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "address": { + "type": "text" + }, + "geopoint": { + "type": "geo_point" + }, + "boundingbox": { + "type": "geo_shape" + }, + "polygon": { + "type": "geo_shape" + }, + "spatialPrecision": { + "type": "text" + }, + "coordinatePrecision": { + "type": "text" + } + } + }, + "temporal": { + "type": "nested", + "properties": { + "from": { + "type": "date", + "format": "year || date_optional_time" + }, + "periodName": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "until": { + "type": "date", + "format": "year || date_optional_time" + }, + "uri": { + "type": "text" + } + } + }, + "nativePeriod": { + "properties": { + "from": { + "type": "date", + "format": "year || date_optional_time" + }, + "periodName": { + "type": "text" + }, + "until": { + "type": "date", + "format": "year || date_optional_time" + } + } + }, + "title": { + "type": "text" + }, + "wasCreated": { + "type": "date", + "format": "year || date_optional_time" + }, + "digitalImage": { + "properties": { + "ariadneUri": { + "type": "keyword" + }, + "primary": { + "type": "boolean" + }, + "providerUri": { + "type": "keyword" + } + } + } + } + } +} +' \ No newline at end of file From e91d82f32c6277aeaa51848b43f076bf23b0675c Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 15 Jun 2021 23:34:59 +0200 Subject: [PATCH 04/13] model classes according to new es mapping; creation and indexing of a record with geopoint data --- .../ariadneplus/elasticsearch/BulkUpload.java | 75 ++-- .../elasticsearch/model/AatSubject.java | 39 -- .../elasticsearch/model/AgentInfo.java | 56 +-- .../model/ArchaeologicalResourceType.java | 56 --- .../model/AriadneCatalogEntry.java | 393 ------------------ .../elasticsearch/model/AriadneGeoPoint.java | 3 - .../elasticsearch/model/AriadneGeoShape.java | 46 ++ .../elasticsearch/model/AriadnePlusEntry.java | 296 +++++++++++++ .../elasticsearch/model/AriadneSubject.java | 36 ++ .../elasticsearch/model/AriadneTemporal.java | 84 ---- .../elasticsearch/model/AriadneURI.java | 31 ++ .../elasticsearch/model/DerivedSubject.java | 20 +- .../elasticsearch/model/Description.java | 31 ++ .../ariadneplus/elasticsearch/model/Dex.java | 31 -- .../elasticsearch/model/DigitalImage.java | 41 ++ .../elasticsearch/model/Distribution.java | 69 --- .../model/ItemMetadataStructure.java | 22 - .../elasticsearch/model/MetadataRecord.java | 33 -- .../elasticsearch/model/NativePeriod.java | 41 ++ .../elasticsearch/model/NativeSubject.java | 2 - .../elasticsearch/model/Spatial.java | 115 ++--- .../elasticsearch/model/Temporal.java | 61 +++ .../ariadneplus/reader/ResourceManager.java | 2 +- .../src/main/resources/application.properties | 49 +-- .../GraphDbReaderAndESIndexTest.java | 3 +- 25 files changed, 738 insertions(+), 897 deletions(-) delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AatSubject.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoShape.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneSubject.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneURI.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Description.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Dex.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DigitalImage.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Distribution.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ItemMetadataStructure.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/MetadataRecord.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativePeriod.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Temporal.java diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 4143947..041683a 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -12,13 +12,13 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.xcontent.XContentType; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.*; import java.util.stream.Collectors; @@ -58,7 +58,7 @@ public class BulkUpload { try { Object next = manager.next(); - AriadneCatalogEntry ace = ((AriadneCatalogEntry) next); + AriadnePlusEntry ace = ((AriadnePlusEntry) next); if (isCollection) { ace.setResourceType("collection"); if (ace.getSpatial()==null) { @@ -68,16 +68,26 @@ public class BulkUpload { else { ace.setResourceType("dataset"); if (ace.getSpatial()!=null) { + ace.getSpatial() + .stream() + .filter(s -> Objects.nonNull(s.getLat()) && Objects.nonNull(s.getLon())) + .forEach(s -> { + double lat = Double.parseDouble(s.getLat()); + double lon = Double.parseDouble(s.getLon()); + GeoPoint geopoint = new GeoPoint(lat, lon); + s.setGeopoint(geopoint); + }); + // TODO update following check according to new model definition if (ace.getSpatial().size()==2) { Spatial uniqueSpatial = new Spatial(); boolean uniquePlaceNameFound = ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).count()==1; - boolean uniqueLocationFound = ace.getSpatial().stream().filter(s -> s.getLocation()!=null).count()==1; + boolean uniqueLocationFound = ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).count()==1; if (uniquePlaceNameFound&&uniqueLocationFound) { ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).forEach(s -> { uniqueSpatial.setPlaceName(s.getPlaceName()); }); - ace.getSpatial().stream().filter(s -> s.getLocation()!=null).forEach(s -> { - uniqueSpatial.setLocation(s.getLocation()); + ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).forEach(s -> { + uniqueSpatial.setGeopoint(s.getGeopoint()); }); ace.getSpatial().clear(); ace.setSpatial(Arrays.asList(uniqueSpatial)); @@ -98,9 +108,10 @@ public class BulkUpload { ace.getSpatial().clear(); ace.setSpatial(dedupSpatials); - if (ace.getSpatial().size()>1) { - ace.getSpatial().removeIf(s -> (s.getPlaceName()!=null&&s.getPlaceName().equals("Name not provided")&&Objects.isNull(s.getLocation()))); - } + // TODO update following check according to new model definition +// if (ace.getSpatial().size()>1) { +// ace.getSpatial().removeIf(s -> (s.getPlaceName()!=null&&s.getPlaceName().equals("Name not provided")&&Objects.isNull(s.getLocation()))); +// } } else { ace.setSpatial(Arrays.asList(new Spatial())); @@ -124,41 +135,19 @@ public class BulkUpload { ace.getContributor().clear(); ace.setContributor(ace.getCreator()); } -// Distribution distribution = new Distribution(); -// AgentInfo distrPublisher = new AgentInfo(); -// distrPublisher.setEmail(""); -// distrPublisher.setName(""); -// distrPublisher.setType(""); -// distribution.setPublisher(Arrays.asList(distrPublisher)); -// ace.setDistribution(Arrays.asList(distribution)); -// ItemMetadataStructure ims = new ItemMetadataStructure(); -// ace.setHasItemMetadataStructure(Arrays.asList(ims)); -// MetadataRecord mr = new MetadataRecord(); -// Dex dex = new Dex(); -// mr.setConformsTo(Arrays.asList(dex)); -// ace.setHasMetadataRecord(Arrays.asList(mr)); -// if (!isCollection) { -// ace.setKeyword(Arrays.asList(new String(""))); -// } -// AgentInfo sr = new AgentInfo(); -// ace.setScientificResponsible(Arrays.asList(sr)); -// AgentInfo tr = new AgentInfo(); -// ace.setTechnicalResponsible(Arrays.asList(tr)); } -// AgentInfo testPublisher = new AgentInfo(); -// testPublisher.setName("TEST"); -// ace.getPublisher().add(testPublisher); + String[] splits = ace.getIdentifier().split("/"); - if (ace.getAatSubjects() != null && ace.getDerivedSubject() != null) { - String aatSource = ace.getAatSubjects().get(0).getId(); - ace.getDerivedSubject().forEach(d -> { - d.setSource(aatSource); - }); - String [] aatSourceSplit = aatSource.split("/"); - String aatSubjectId = aatSourceSplit[aatSourceSplit.length-1]; - ace.getAatSubjects().forEach(s -> s.setId(aatSubjectId)); - } +// if (ace.getAatSubjects() != null && ace.getDerivedSubject() != null) { +// String aatSource = ace.getAatSubjects().get(0).getId(); +// ace.getDerivedSubject().forEach(d -> { +// d.setSource(aatSource); +// }); +// String [] aatSourceSplit = aatSource.split("/"); +// String aatSubjectId = aatSourceSplit[aatSourceSplit.length-1]; +// ace.getAatSubjects().forEach(s -> s.setId(aatSubjectId)); +// } String idES = splits[splits.length-1]; request.add(new IndexRequest(elasticSearchIndexName).id(idES) @@ -209,9 +198,9 @@ public class BulkUpload { } String lat = ""; String lon = ""; - if (!Objects.isNull(spatial.getLocation())) { - lat = Float.toString(spatial.getLocation().getLat()); - lon = Float.toString(spatial.getLocation().getLon()); + if (!Objects.isNull(spatial.getGeopoint())) { + lat = Double.toString(spatial.getGeopoint().getLat()); + lon = Double.toString(spatial.getGeopoint().getLon()); } String uniqueAttribute = (name) + (lat) + (lon); return uniqueAttribute; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AatSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AatSubject.java deleted file mode 100644 index 1157902..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AatSubject.java +++ /dev/null @@ -1,39 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; -import com.google.gson.*; - -public class AatSubject { - private String id; - private String label; - private String lang; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getLang() { - return lang; - } - - public void setLang(String lang) { - this.lang = lang; - } - - public AatSubject() { - } - - public static AatSubject fromJson(String json){ - return new Gson().fromJson(json, AatSubject.class); - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java index 349c89c..43f17d5 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java @@ -1,15 +1,13 @@ package eu.dnetlib.ariadneplus.elasticsearch.model; import com.google.gson.Gson; -import com.google.gson.JsonElement; - -import java.util.Map; public class AgentInfo { private String email = new String(""); - private String name = new String(""); - private String phone = new String(""); - private String type = new String(""); + private String homepage = new String(""); + private String institution = new String(""); + private String name = new String(""); + private String agentIdentifier = new String(""); public AgentInfo() { @@ -31,45 +29,31 @@ public class AgentInfo { this.name = name; } - public String getPhone() { - return phone; + public String getHomepage() { + return homepage; } - public void setPhone(String phone) { - this.phone = phone; + public void setHomepage(String homepage) { + this.homepage = homepage; } - public String getType() { - return type; + public String getInstitution() { + return institution; } - public void setType(String type) { - this.type = type; + public void setInstitution(String institution) { + this.institution = institution; + } + + public String getAgentIdentifier() { + return agentIdentifier; + } + + public void setAgentIdentifier(String agentIdentifier) { + this.agentIdentifier = agentIdentifier; } public static AgentInfo fromJson(String json){ return new Gson().fromJson(json, AgentInfo.class); } - - public static AgentInfo fromRDFJson(JsonElement json){ - AgentInfo pi = new AgentInfo(); - for (Map.Entry entry : json.getAsJsonObject().entrySet()){ - switch (entry.getKey()){ - case "https://www.ariadne-infrastructure.eu/property/name" : - pi.setName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "https://www.ariadne-infrastructure.eu/property/type": - pi.setType(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "https://www.ariadne-infrastructure.eu/property/email": - pi.setEmail(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "https://www.ariadne-infrastructure.eu/property/phone": - pi.setPhone(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - } - } - - return pi; - } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java deleted file mode 100644 index 7c2a527..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java +++ /dev/null @@ -1,56 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; -import com.google.gson.JsonElement; - -import java.util.HashMap; -import java.util.Map; - -public class ArchaeologicalResourceType { - private long id = -1; - private String name; - - private transient HashMap type =new HashMap<>(); - - public ArchaeologicalResourceType() { - type.put("Site/monument", new Long(10)); - type.put("Fieldwork", new Long(11)); - type.put("Fieldwork report", new Long(12)); - type.put("Scientific analysis", new Long(13)); - type.put("Date", new Long(14)); - type.put("Artefact", new Long(15)); - type.put("Fieldwork archive", new Long(16)); - type.put("Inscription", new Long(17)); - type.put("Burial", new Long(18)); - type.put("Rock Art", new Long(19)); - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - if (this.name==null) { - setId(-1); - } - if (type.containsKey(this.name)) { - setId(type.get(name).longValue()); - } - else { - setId(-2); - } - } - - public static ArchaeologicalResourceType fromJson(String json){ - return new Gson().fromJson(json, ArchaeologicalResourceType.class); - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java deleted file mode 100644 index 2d802c4..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java +++ /dev/null @@ -1,393 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; -import eu.dnetlib.ariadneplus.reader.utils.ESUtils; - -import java.util.List; - -public class AriadneCatalogEntry { - private List aatSubjects; - private String accessPolicy; - private String accessRights; - private ArchaeologicalResourceType archaeologicalResourceType; - private List contributor; - private List creator; - private List derivedSubject; - private String description; - private List distribution; - private List publisher; - private String title; - private List hasItemMetadataStructure; - private List hasMetadataRecord; - private String identifier; - private List isPartOf; - private transient String uniqueIsPartOf; - private String issued; - private List keyword; - private String landingPage; - private String language; - private List legalResponsible; - private String modified; - private List nativeSubject; - private String originalId; - private List owner; - private String resourceType; - private String rdfType; - private List scientificResponsible; - private List spatial; -// private List spatialRegion; -// private List spatialRegionPoint; - private List technicalResponsible; - private List temporal; - - private String accrualPeriodicity;// = new String(""); - private String audience;// IF PRESENT, MUST CONTAIN A VALUE !! - private String contactPoint;// = new String(""); - private String extent;// IF PRESENT, MUST CONTAIN A VALUE !! - private String providerId;// = new String(""); - private String packageId;// = new String(""); - private String placeName;// = new String(""); - private String postcode;// = new String(""); - private String rdfAbout;// = new String(""); - private String rights;// = new String(""); - - public List getContributor() { - return contributor; - } - - public void setContributor(List contributor) { - this.contributor = contributor; - } - - - - public List getAatSubjects() { - return aatSubjects; - } - - public void setAatSubjects(List aatSubjects) { - this.aatSubjects = aatSubjects; - } - - public String getAccessPolicy() { - return accessPolicy; - } - - public void setAccessPolicy(String accessPolicy) { - this.accessPolicy = accessPolicy; - } - - public String getAccessRights() { - return accessRights; - } - - public void setAccessRights(String accessRights) { - this.accessRights = accessRights; - } - - public ArchaeologicalResourceType getArchaeologicalResourceType() { - return archaeologicalResourceType; - } - - public void setArchaeologicalResourceType(ArchaeologicalResourceType archaeologicalResourceType) { - this.archaeologicalResourceType = archaeologicalResourceType; - } - - public String getContactPoint() { - return contactPoint; - } - - public void setContactPoint(String contactPoint) { - this.contactPoint = contactPoint; - } - - public List getCreator() { - return creator; - } - - public void setCreator(List creator) { - this.creator = creator; - } - - public List getDerivedSubject() { - return derivedSubject; - } - - public void setDerivedSubject(List derivedSubject) { - this.derivedSubject = derivedSubject; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public List getDistribution() { - return distribution; - } - - public void setDistribution(List distribution) { - this.distribution = distribution; - } - - public List getPublisher() { - return publisher; - } - - public void setPublisher(List publisher) { - this.publisher = publisher; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getExtent() { - return extent; - } - - public void setExtent(String extent) { - this.extent = extent; - } - - public List getHasItemMetadataStructure() { - return hasItemMetadataStructure; - } - - public void setHasItemMetadataStructure(List hasItemMetadataStructure) { - this.hasItemMetadataStructure = hasItemMetadataStructure; - } - - public List getHasMetadataRecord() { - return hasMetadataRecord; - } - - public void setHasMetadataRecord(List hasMetadataRecord) { - this.hasMetadataRecord = hasMetadataRecord; - } - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - public List getIsPartOf() { - return isPartOf; - } - - public void setIsPartOf(List isPartOf) { - this.isPartOf = isPartOf; - } - - public String getIssued() { - return issued; - } - - public void setIssued(String issued) { - this.issued = ESUtils.getESFormatDate(issued); - } - - public List getKeyword() { - return keyword; - } - - public void setKeyword(List keyword) { - this.keyword = keyword; - } - - public String getLandingPage() { - return landingPage; - } - - public void setLandingPage(String landingPage) { - this.landingPage = landingPage; - } - - public String getLanguage() { - return language; - } - - public void setLanguage(String language) { - if (language!=null && language.equals("eng")) { - this.language = "en"; - } - else { - this.language = language; - } - } - - public List getLegalResponsible() { - return legalResponsible; - } - - public void setLegalResponsible(List legalResponsible) { - this.legalResponsible = legalResponsible; - } - - public String getModified() { - return modified; - } - - public void setModified(String modified) { - this.modified = ESUtils.getESFormatDate(modified); - } - - public List getNativeSubject() { - return nativeSubject; - } - - public void setNativeSubject(List nativeSubject) { - this.nativeSubject = nativeSubject; - } - - public String getOriginalId() { - return originalId; - } - - public void setOriginalId(String originalId) { - this.originalId = originalId; - } - - public List getOwner() { - return owner; - } - - public void setOwner(List owner) { - this.owner = owner; - } - - public String getPackageId() { - return packageId; - } - - public void setPackageId(String packageId) { - this.packageId = packageId; - } - - public String getPlaceName() { - return placeName; - } - - public void setPlaceName(String placeName) { - this.placeName = placeName; - } - - public String getPostcode() { - return postcode; - } - - public void setPostcode(String postcode) { - this.postcode = postcode; - } - - public String getProviderId() { - return providerId; - } - - public void setProviderId(String providerId) { - this.providerId = providerId; - } - - public String getRdfAbout() { - return rdfAbout; - } - - public void setRdfAbout(String rdfAbout) { - this.rdfAbout = rdfAbout; - } - - public String getResourceType() { - return resourceType; - } - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - - public String getRights() { - return rights; - } - - public void setRights(String rights) { - this.rights = rights; - } - - public List getScientificResponsible() { - return scientificResponsible; - } - - public void setScientificResponsible(List scientificResponsible) { - this.scientificResponsible = scientificResponsible; - } - - public List getSpatial() { - return spatial; - } - - public void setSpatial(List spatial) { - if (this.spatial==null) { - this.spatial = spatial; - } - else { - this.spatial.addAll(spatial); - } - } - - public List getTechnicalResponsible() { - return technicalResponsible; - } - - public void setTechnicalResponsible(List technicalResponsible) { - this.technicalResponsible = technicalResponsible; - } - - public List getTemporal() { - return temporal; - } - - public void setTemporal(List temporal) { - this.temporal = temporal; - } - - public String getUniqueIsPartOf() { - return uniqueIsPartOf; - } - - public void setUniqueIsPartOf(String uniqueIsPartOf) { - this.uniqueIsPartOf = uniqueIsPartOf; - } - - public static AriadneCatalogEntry fromJson(String json){ - return new Gson().fromJson(json, AriadneCatalogEntry.class); - } - - public String toJson(){ - return new Gson().toJson(this); - } - - public void setAccrualPeriodicity(String accrualPeriodicity) { - this.accrualPeriodicity = accrualPeriodicity; - } - - public void setAudience(String audience) { - this.audience = audience; - } - - public String getRdfType() { - return rdfType; - } - - public void setRdfType(String rdfType) { - this.rdfType = rdfType; - } -} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java index b37cf0c..ec8c075 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java @@ -1,9 +1,6 @@ package eu.dnetlib.ariadneplus.elasticsearch.model; import com.google.gson.Gson; -import com.google.gson.JsonElement; - -import java.util.Map; public class AriadneGeoPoint { private float lat; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoShape.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoShape.java new file mode 100644 index 0000000..34f8d4c --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoShape.java @@ -0,0 +1,46 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class AriadneGeoShape { + private float lat; + private float lon; + + public float getLat() { + return lat; + } + + public void setLat(float lat) { + this.lat = lat; + } + + public float getLon() { + return lon; + } + + public void setLon(float lon) { + this.lon = lon; + } + + public AriadneGeoShape() { + } + + public static AriadneGeoShape fromJson (String json){ + return new Gson().fromJson(json, AriadneGeoShape.class); + } + +// public static AriadneGeoPoint fromRDFJson(JsonElement json){ +// AriadneGeoPoint agp = new AriadneGeoPoint(); +// for (Map.Entry stringJsonElementEntry : json.getAsJsonObject().entrySet()) { +// switch (stringJsonElementEntry.getKey()){ +// case "https://www.ariadne-infrastructure.eu/property/lat": +// agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "https://www.ariadne-infrastructure.eu/property/lon": +// agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// } +// } +// return agp; +// } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java new file mode 100644 index 0000000..c09cc74 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java @@ -0,0 +1,296 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import eu.dnetlib.ariadneplus.reader.utils.ESUtils; + +import java.util.List; + +public class AriadnePlusEntry { + private List derivedSubject; + private String accessPolicy; + private String accessRights; + private List ariadneSubject; + private List contributor; + private List creator; + private Description description; + private String extent; + private String identifier; + private List isPartOf; + private String issued; + private String landingPage; + private String language; + private String modified; + private List nativeSubject; + private String originalId; + private List owner; + private List publisher; + private AriadneURI is_about; + private String resourceType; + private AriadneURI has_type; + private List responsible; + private List spatial; + private List temporal; + private String title; + private NativePeriod nativePeriod; + private String wasCreated; + private DigitalImage digitalImage; + + private transient String uniqueIsPartOf; + + public List getContributor() { + return contributor; + } + + public void setContributor(List contributor) { + this.contributor = contributor; + } + + public String getAccessPolicy() { + return accessPolicy; + } + + public void setAccessPolicy(String accessPolicy) { + this.accessPolicy = accessPolicy; + } + + public String getAccessRights() { + return accessRights; + } + + public void setAccessRights(String accessRights) { + this.accessRights = accessRights; + } + + public List getCreator() { + return creator; + } + + public void setCreator(List creator) { + this.creator = creator; + } + + public List getDerivedSubject() { + return derivedSubject; + } + + public void setDerivedSubject(List derivedSubject) { + this.derivedSubject = derivedSubject; + } + + public List getPublisher() { + return publisher; + } + + public void setPublisher(List publisher) { + this.publisher = publisher; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getExtent() { + return extent; + } + + public void setExtent(String extent) { + this.extent = extent; + } + + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public List getIsPartOf() { + return isPartOf; + } + + public void setIsPartOf(List isPartOf) { + this.isPartOf = isPartOf; + } + + public String getIssued() { + return issued; + } + + public void setIssued(String issued) { + this.issued = ESUtils.getESFormatDate(issued); + } + + public String getLandingPage() { + return landingPage; + } + + public void setLandingPage(String landingPage) { + this.landingPage = landingPage; + } + + public String getLanguage() { + return language; + } + + public void setLanguage(String language) { + if (language!=null && language.equals("eng")) { + this.language = "en"; + } + else { + this.language = language; + } + } + + public String getModified() { + return modified; + } + + public void setModified(String modified) { + this.modified = ESUtils.getESFormatDate(modified); + } + + public List getNativeSubject() { + return nativeSubject; + } + + public void setNativeSubject(List nativeSubject) { + this.nativeSubject = nativeSubject; + } + + public String getOriginalId() { + return originalId; + } + + public void setOriginalId(String originalId) { + this.originalId = originalId; + } + + public List getOwner() { + return owner; + } + + public void setOwner(List owner) { + this.owner = owner; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + + public List getSpatial() { + return spatial; + } + + public void setSpatial(List spatial) { + if (this.spatial==null) { + this.spatial = spatial; + } + else { + this.spatial.addAll(spatial); + } + } + + public List getTemporal() { + return temporal; + } + + public void setTemporal(List temporal) { + this.temporal = temporal; + } + + public void setDescription(Description description) { + this.description = description; + } + + public void setIs_about(AriadneURI is_about) { + this.is_about = is_about; + } + + public void setHas_type(AriadneURI has_type) { + this.has_type = has_type; + } + + public void setResponsible(List responsible) { + this.responsible = responsible; + } + + public void setWasCreated(String wasCreated) { + this.wasCreated = wasCreated; + } + + public String getUniqueIsPartOf() { + return uniqueIsPartOf; + } + + public void setUniqueIsPartOf(String uniqueIsPartOf) { + this.uniqueIsPartOf = uniqueIsPartOf; + } + + public static AriadnePlusEntry fromJson(String json){ + return new Gson().fromJson(json, AriadnePlusEntry.class); + } + + public Description getDescription() { + return description; + } + + public AriadneURI getIs_about() { + return is_about; + } + + public AriadneURI getHas_type() { + return has_type; + } + + public List getResponsible() { + return responsible; + } + + public String getWasCreated() { + return wasCreated; + } + + public List getAriadneSubject() { + return ariadneSubject; + } + + public void setAriadneSubject(List ariadneSubject) { + this.ariadneSubject = ariadneSubject; + } + + public NativePeriod getNativePeriod() { + return nativePeriod; + } + + public void setNativePeriod(NativePeriod nativePeriod) { + this.nativePeriod = nativePeriod; + } + + public DigitalImage getDigitalImage() { + return digitalImage; + } + + public void setDigitalImage(DigitalImage digitalImage) { + this.digitalImage = digitalImage; + } + + public String toJson(){ + return new Gson().toJson(this); + } + + public void setDescription(String description) { + Description descr = new Description(); + descr.setText(description); + this.description = descr; + } +} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneSubject.java new file mode 100644 index 0000000..138e52b --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneSubject.java @@ -0,0 +1,36 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +import java.util.HashMap; + +public class AriadneSubject { + private String prefLabel; + + public String getPrefLabel() { + return prefLabel; + } + + public void setPrefLabel(String prefLabel) { + this.prefLabel = prefLabel; + } + + private transient HashMap typeValues =new HashMap<>(); + + public AriadneSubject() { + typeValues.put("Site/monument", new Long(10)); + typeValues.put("Fieldwork", new Long(11)); + typeValues.put("Fieldwork report", new Long(12)); + typeValues.put("Scientific analysis", new Long(13)); + typeValues.put("Date", new Long(14)); + typeValues.put("Artefact", new Long(15)); + typeValues.put("Fieldwork archive", new Long(16)); + typeValues.put("Inscription", new Long(17)); + typeValues.put("Burial", new Long(18)); + typeValues.put("Rock Art", new Long(19)); + } + + public static AriadneSubject fromJson(String json){ + return new Gson().fromJson(json, AriadneSubject.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java deleted file mode 100644 index 4263fbb..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java +++ /dev/null @@ -1,84 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; -import com.google.gson.JsonElement; - -import java.util.Map; - -public class AriadneTemporal { - private String from; - private String periodName; - private String until; - private String uri; - private transient String matchingPeriodOName; - - public static AriadneTemporal fromRDFJson(JsonElement json) { - AriadneTemporal at = new AriadneTemporal(); - for (Map.Entry entry : json.getAsJsonObject().entrySet()){ - switch (entry.getKey()){ - case "https://www.ariadne-infrastructure.eu/property/from" : - at.setFrom(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "https://www.ariadne-infrastructure.eu/property/periodName": - at.setPeriodName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "https://www.ariadne-infrastructure.eu/property/until": - at.setUntil(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "https://www.ariadne-infrastructure.eu/property/uri": - at.setUri(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - - } - } - - return at; - } - - public String getFrom() { - return from; - } - - public void setFrom(String from) { - this.from = from; - } - - public String getPeriodName() { - return periodName; - } - - public void setPeriodName(String periodName) { - this.periodName = periodName; - } - - public String getUntil() { - return until; - } - - public void setUntil(String until) { - this.until = until; - } - - public String getUri() { - return uri; - } - - public void setUri(String uri) { - this.uri = uri; - } - - public AriadneTemporal() { - } - - public static AriadneTemporal fromJson(String json){ - return new Gson().fromJson(json, AriadneTemporal.class); - } - - public String getMatchingPeriodOName() { - return matchingPeriodOName; - } - - public void setMatchingPeriodOName(String matchingPeriodOName) { - this.matchingPeriodOName = matchingPeriodOName; - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneURI.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneURI.java new file mode 100644 index 0000000..c5b67a3 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneURI.java @@ -0,0 +1,31 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class AriadneURI { + private String label; + private String uri; + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public AriadneURI() { + } + + public static AriadneURI fromJson(String json){ + return new Gson().fromJson(json, AriadneURI.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DerivedSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DerivedSubject.java index 2ea7221..22eb49e 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DerivedSubject.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DerivedSubject.java @@ -4,7 +4,9 @@ import com.google.gson.Gson; public class DerivedSubject { private String prefLabel; - private String source; + private String source; + private String id; + private String lang; public String getPrefLabel() { return prefLabel; @@ -22,6 +24,22 @@ public class DerivedSubject { this.source = source; } + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getLang() { + return lang; + } + + public void setLang(String lang) { + this.lang = lang; + } + public DerivedSubject() { } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Description.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Description.java new file mode 100644 index 0000000..fb252af --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Description.java @@ -0,0 +1,31 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class Description { + private String text; + private String language; + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public String getLanguage() { + return language; + } + + public void setLanguage(String language) { + this.language = language; + } + + public Description() { + } + + public static Description fromJson(String json){ + return new Gson().fromJson(json, Description.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Dex.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Dex.java deleted file mode 100644 index 8efc857..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Dex.java +++ /dev/null @@ -1,31 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; - -public class Dex { - private String characterSet = new String(""); - private String description = new String(""); - - public String getCharacterSet() { - return characterSet; - } - - public void setCharacterSet(String characterSet) { - this.characterSet = characterSet; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Dex() { - } - - public static Dex fromJson(String json){ - return new Gson().fromJson(json,Dex.class); - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DigitalImage.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DigitalImage.java new file mode 100644 index 0000000..79107fe --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DigitalImage.java @@ -0,0 +1,41 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class DigitalImage { + + private String ariadneUri; + private String primary; + private String providerUri; + + public String getAriadneUri() { + return ariadneUri; + } + + public void setAriadneUri(String ariadneUri) { + this.ariadneUri = ariadneUri; + } + + public String getPrimary() { + return primary; + } + + public void setPrimary(String primary) { + this.primary = primary; + } + + public String getProviderUri() { + return providerUri; + } + + public void setProviderUri(String providerUri) { + this.providerUri = providerUri; + } + + public DigitalImage() { + } + + public static DigitalImage fromJson(String json){ + return new Gson().fromJson(json, DigitalImage.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Distribution.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Distribution.java deleted file mode 100644 index 3c86a89..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Distribution.java +++ /dev/null @@ -1,69 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; - -import java.util.List; - -public class Distribution { - private String accessURL = new String(""); - private String description = new String(""); - private String issued = new String("9999"); - private String modified = new String("9999"); - private List publisher; - private String title = new String(""); - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public List getPublisher() { - return publisher; - } - - public void setPublisher(List publisher) { - this.publisher = publisher; - } - - public String getAccessURL() { - return accessURL; - } - - public void setAccessURL(String accessURL) { - this.accessURL = accessURL; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getIssued() { - return issued; - } - - public void setIssued(String issued) { - this.issued = issued; - } - - public String getModified() { - return modified; - } - - public void setModified(String modified) { - this.modified = modified; - } - - public Distribution() { - } - - public static Distribution fromJson(String json){ - return new Gson().fromJson(json,Distribution.class); - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ItemMetadataStructure.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ItemMetadataStructure.java deleted file mode 100644 index fe28cf3..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ItemMetadataStructure.java +++ /dev/null @@ -1,22 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; - -public class ItemMetadataStructure { - private String characterSet = new String(""); - - public String getCharacterSet() { - return characterSet; - } - - public void setCharacterSet(String characterSet) { - this.characterSet = characterSet; - } - - public ItemMetadataStructure() { - } - - private static ItemMetadataStructure fromJson(String json){ - return new Gson().fromJson(json, ItemMetadataStructure.class); - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/MetadataRecord.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/MetadataRecord.java deleted file mode 100644 index dcd9946..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/MetadataRecord.java +++ /dev/null @@ -1,33 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; - -import java.util.List; - -public class MetadataRecord { - private List conformsTo; - private String xmlDoc = new String(""); - - public List getConformsTo() { - return conformsTo; - } - - public void setConformsTo(List conformsTo) { - this.conformsTo = conformsTo; - } - - public String getXmlDoc() { - return xmlDoc; - } - - public void setXmlDoc(String xmlDoc) { - this.xmlDoc = xmlDoc; - } - - public MetadataRecord() { - } - - public static MetadataRecord fromJson(String json){ - return new Gson().fromJson(json, MetadataRecord.class); - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativePeriod.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativePeriod.java new file mode 100644 index 0000000..38b7ec7 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativePeriod.java @@ -0,0 +1,41 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class NativePeriod { + private String from; + private String periodName; + private String until; + + public String getFrom() { + return from; + } + + public void setFrom(String from) { + this.from = from; + } + + public String getPeriodName() { + return periodName; + } + + public void setPeriodName(String periodName) { + this.periodName = periodName; + } + + public String getUntil() { + return until; + } + + public void setUntil(String until) { + this.until = until; + } + + public NativePeriod() { + } + + public static NativePeriod fromJson(String json){ + return new Gson().fromJson(json, NativePeriod.class); + } + +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java index a211d37..2da5356 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java @@ -9,8 +9,6 @@ public class NativeSubject { private String prefLabel; private String rdfAbout; - - public String getPrefLabel() { return prefLabel; } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java index e635161..31a34d0 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java @@ -1,24 +1,25 @@ package eu.dnetlib.ariadneplus.elasticsearch.model; import com.google.gson.Gson; -import com.google.gson.JsonElement; - -import java.util.Map; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.elasticsearch.common.geo.GeoPoint; public class Spatial { - private String address;// = new String(""); - private String boundingBoxMaxLat; - private String boundingBoxMaxLon; - private String boundingBoxMinLat; - private String boundingBoxMinLon; - private String coordinateSystem = new String(""); - private String country;// = new String(""); - private AriadneGeoPoint location; private String placeName; - private String lat; - private String lon; - private String postcode = new String(""); + private String address; + private GeoPoint geopoint; + private GeoShape boundingbox; + private GeoShape polygon; + private String spatialPrecision; + + private transient String coordinatePrecision; + private transient String boundingBoxMaxLat; + private transient String boundingBoxMaxLon; + private transient String boundingBoxMinLat; + private transient String boundingBoxMinLon; + private transient String lat; + private transient String lon; public String getAddress() { return address; @@ -60,30 +61,6 @@ public class Spatial { this.boundingBoxMinLon = boundingBoxMinLon; } - public String getCoordinateSystem() { - return coordinateSystem; - } - - public void setCoordinateSystem(String coordinateSystem) { - this.coordinateSystem = coordinateSystem; - } - - public String getCountry() { - return country; - } - - public void setCountry(String country) { - this.country = country; - } - - public AriadneGeoPoint getLocation() { - return location; - } - - public void setLocation(AriadneGeoPoint location) { - this.location = location; - } - public String getPlaceName() { return placeName; } @@ -92,22 +69,60 @@ public class Spatial { this.placeName = placeName; } + public GeoShape getBoundingbox() { + return boundingbox; + } + + public void setBoundingbox(GeoShape boundingbox) { + this.boundingbox = boundingbox; + } + + public GeoShape getPolygon() { + return polygon; + } + + public void setPolygon(GeoShape polygon) { + this.polygon = polygon; + } + + public String getSpatialPrecision() { + return spatialPrecision; + } + + public void setSpatialPrecision(String spatialPrecision) { + this.spatialPrecision = spatialPrecision; + } + + public String getCoordinatePrecision() { + return coordinatePrecision; + } + + public void setCoordinatePrecision(String coordinatePrecision) { + this.coordinatePrecision = coordinatePrecision; + } + + public String getLat() { + return lat; + } + public void setLat(String lat) { - if (this.getLocation()==null) { - this.setLocation(new AriadneGeoPoint()); - } - if (lat!=null) { - this.getLocation().setLat(Float.parseFloat(lat)); - } + this.lat = lat; + } + + public String getLon() { + return lon; } public void setLon(String lon) { - if (this.getLocation()==null) { - this.setLocation(new AriadneGeoPoint()); - } - if (lon!=null) { - this.getLocation().setLon(Float.parseFloat(lon)); - } + this.lon = lon; + } + + public GeoPoint getGeopoint() { + return geopoint; + } + + public void setGeopoint(GeoPoint geopoint) { + this.geopoint = geopoint; } public Spatial() { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Temporal.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Temporal.java new file mode 100644 index 0000000..020e894 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Temporal.java @@ -0,0 +1,61 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + +public class Temporal { + private String from; + private String periodName; + private String until; + private String uri; + private transient String matchingPeriodOName; + + public String getFrom() { + return from; + } + + public void setFrom(String from) { + this.from = from; + } + + public String getPeriodName() { + return periodName; + } + + public void setPeriodName(String periodName) { + this.periodName = periodName; + } + + public String getUntil() { + return until; + } + + public void setUntil(String until) { + this.until = until; + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public Temporal() { + } + + public static Temporal fromJson(String json){ + return new Gson().fromJson(json, Temporal.class); + } + + public String getMatchingPeriodOName() { + return matchingPeriodOName; + } + + public void setMatchingPeriodOName(String matchingPeriodOName) { + this.matchingPeriodOName = matchingPeriodOName; + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java index 5072839..502e80d 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java @@ -90,7 +90,7 @@ public class ResourceManager { class_name = (String)tmp.get("value"); //TODO: Use rdf:type instead of these values that are added statically by the CONSTRUCT queries (that need to be changed as well to include the rdf:type if (class_name.equals("Record") || class_name.equals("Collection")) { - class_name = "AriadneCatalogEntry"; + class_name = "AriadnePlusEntry"; } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 8ff8ffb..23926db 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -18,7 +18,7 @@ general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. type.path=https://www.ariadne-infrastructure.eu/property/rdfType exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "https://www.ariadne-infrastructure.eu/property/rdfType"] class.map.specifications={\ -"AriadneTemporal": {\ +"Temporal": {\ "class_type": "prototype",\ "mappings": {\ "https://www.ariadne-infrastructure.eu/property/from": {\ @@ -121,16 +121,11 @@ class.map.specifications={\ "element_type": "java.lang.String"\ }}\ },\ -"ArchaeologicalResourceType": {\ -"class_type": "unique",\ +"AriadneSubject": {\ +"class_type": "prototype",\ "mappings": {\ -"https://www.ariadne-infrastructure.eu/property/id": {\ -"class_field": "Id",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ "https://www.ariadne-infrastructure.eu/property/name": {\ -"class_field": "Name",\ +"class_field": "PrefLabel",\ "substring": "no",\ "element_type": "java.lang.String"\ }\ @@ -154,8 +149,13 @@ class.map.specifications={\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"https://www.ariadne-infrastructure.eu/property/phone": {\ -"class_field": "Phone",\ +"https://www.ariadne-infrastructure.eu/property/homepage": {\ +"class_field": "Homepage",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/institution": {\ +"class_field": "Institution",\ "substring": "no",\ "element_type": "java.lang.String"\ }\ @@ -188,19 +188,14 @@ class.map.specifications={\ "class_field": "Source",\ "substring": "yes",\ "element_type": "java.lang.String"\ -}\ -}\ },\ -"AatSubject": {\ -"class_type": "prototype",\ -"mappings": {\ "https://www.ariadne-infrastructure.eu/property/id": {\ "class_field": "Id",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ "https://www.ariadne-infrastructure.eu/property/label": {\ -"class_field": "Label",\ +"class_field": "PrefLabel",\ "substring": "yes",\ "element_type": "java.lang.String"\ },\ @@ -211,7 +206,7 @@ class.map.specifications={\ }\ }\ },\ -"AriadneCatalogEntry": {\ +"AriadnePlusEntry": {\ "class_type": "unique",\ "mappings": {\ "https://www.ariadne-infrastructure.eu/property/accessPolicy": {\ @@ -230,8 +225,8 @@ class.map.specifications={\ "substring": "no"\ },\ "https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {\ -"class_field": "ArchaeologicalResourceType",\ -"external_reference": "ArchaeologicalResourceType",\ +"class_field": "AriadneSubject",\ +"external_reference": "AriadneSubject",\ "substring": "no"\ },\ "https://www.ariadne-infrastructure.eu/property/issued": {\ @@ -259,11 +254,6 @@ class.map.specifications={\ "substring": "no",\ "external_reference": "DerivedSubject"\ },\ -"https://www.ariadne-infrastructure.eu/property/aatSubjects": {\ -"class_field": "AatSubjects",\ -"substring": "no",\ -"external_reference": "AatSubject"\ -},\ "https://www.ariadne-infrastructure.eu/property/spatialRegion": {\ "class_field": "Spatial",\ "substring": "no",\ @@ -284,11 +274,6 @@ class.map.specifications={\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"https://www.ariadne-infrastructure.eu/property/placeName": {\ -"class_field": "PlaceName",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ "https://www.ariadne-infrastructure.eu/property/title": {\ "class_field": "Title",\ "substring": "no",\ @@ -315,7 +300,7 @@ class.map.specifications={\ "external_reference": "AgentInfo"\ },\ "https://www.ariadne-infrastructure.eu/property/legalResponsible": {\ -"class_field": "LegalResponsible",\ +"class_field": "Responsible",\ "substring": "no",\ "external_reference": "AgentInfo"\ },\ @@ -332,7 +317,7 @@ class.map.specifications={\ "https://www.ariadne-infrastructure.eu/property/temporal": {\ "class_field": "Temporal",\ "substring": "no",\ -"external_reference": "AriadneTemporal"\ +"external_reference": "Temporal"\ },\ "https://www.ariadne-infrastructure.eu/property/language": {\ "class_field": "Language",\ diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index adac55a..8835c50 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -72,7 +72,6 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore public void uploadADSRecordTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/34E3811A-0BAD-3832-B3A0-3139E8A0285C"; @@ -192,7 +191,7 @@ public class GraphDbReaderAndESIndexTest { } @Test -// @Ignore + @Ignore public void uploadROADTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FC70B370-C489-31C5-B1D4-339CFD28CF2B"; From 03da34a897a016abb8145b80f8f13327a3dc0151 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 15 Jun 2021 23:38:02 +0200 Subject: [PATCH 05/13] added dynamic property set to false to avoid es mapping modification indexinga record; previous index has been deleted --- .../src/main/resources/es/schemaAriadneplusElasticsearch.curl | 1 + 1 file changed, 1 insertion(+) diff --git a/dnet-ariadneplus/src/main/resources/es/schemaAriadneplusElasticsearch.curl b/dnet-ariadneplus/src/main/resources/es/schemaAriadneplusElasticsearch.curl index 2a04585..1721fba 100644 --- a/dnet-ariadneplus/src/main/resources/es/schemaAriadneplusElasticsearch.curl +++ b/dnet-ariadneplus/src/main/resources/es/schemaAriadneplusElasticsearch.curl @@ -1,6 +1,7 @@ curl -X PUT "http://elastic-test.ariadne.d4science.org:9200/ariadneplus?pretty" -H 'Content-Type: application/json' -d ' { "mappings": { + "dynamic": false, "properties": { "derivedSubject": { "properties": { From 8ed4b8b08d9266b9579367720b7366f9ecf22903 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Thu, 17 Jun 2021 17:59:51 +0200 Subject: [PATCH 06/13] added spatial boundingbox support using a 4 sides polygon wkt format --- .../ariadneplus/elasticsearch/BulkUpload.java | 64 +++++++++++++------ .../elasticsearch/model/AriadneGeoPoint.java | 46 ------------- .../elasticsearch/model/AriadneGeoShape.java | 46 ------------- .../elasticsearch/model/AriadnePlusEntry.java | 18 +++--- .../elasticsearch/model/Spatial.java | 36 +++++------ .../elasticsearch/model/Temporal.java | 9 --- .../read_collection_data_template.sparql | 2 +- .../sparql/read_record_data_template.sparql | 2 +- .../GraphDbReaderAndESIndexTest.java | 2 +- 9 files changed, 75 insertions(+), 150 deletions(-) delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java delete mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoShape.java diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 041683a..5732aee 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -6,19 +6,28 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpHost; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.GeoPolygon; +import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; +import org.apache.lucene.spatial3d.geom.PlanetModel; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.builders.CoordinatesBuilder; +import org.elasticsearch.common.geo.builders.EnvelopeBuilder; +import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.geometry.Polygon; +import org.locationtech.jts.geom.Coordinate; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import java.io.IOException; +import java.io.OutputStream; import java.util.*; import java.util.stream.Collectors; @@ -74,10 +83,35 @@ public class BulkUpload { .forEach(s -> { double lat = Double.parseDouble(s.getLat()); double lon = Double.parseDouble(s.getLon()); - GeoPoint geopoint = new GeoPoint(lat, lon); + org.elasticsearch.common.geo.GeoPoint geopoint = new org.elasticsearch.common.geo.GeoPoint(lat, lon); s.setGeopoint(geopoint); }); - // TODO update following check according to new model definition + ace.getSpatial() + .stream() + .filter(s -> Objects.nonNull(s.getBoundingBoxMaxLat()) + && Objects.nonNull(s.getBoundingBoxMaxLon()) + && Objects.nonNull(s.getBoundingBoxMinLat()) + && Objects.nonNull(s.getBoundingBoxMinLon())) + .forEach(s -> { + double maxlat = Double.parseDouble(s.getBoundingBoxMaxLat()); + double minlat = Double.parseDouble(s.getBoundingBoxMinLat()); + double minlon = Double.parseDouble(s.getBoundingBoxMinLon()); + double maxlon = Double.parseDouble(s.getBoundingBoxMaxLon()); + CoordinatesBuilder coordinatesBuilder = new CoordinatesBuilder(); + coordinatesBuilder.coordinate(minlon, maxlat); + coordinatesBuilder.coordinate(minlon, minlat); + coordinatesBuilder.coordinate(maxlon, minlat); + coordinatesBuilder.coordinate(maxlon, maxlat); + coordinatesBuilder.coordinate(minlon, maxlat); + PolygonBuilder polygonBuilder = new PolygonBuilder(coordinatesBuilder); + String wkt = polygonBuilder.toWKT(); + s.setBoundingbox(wkt); +// Coordinate topLeft = new Coordinate(minlon, maxlat); +// Coordinate bottomRight = new Coordinate(maxlon, minlat); +// EnvelopeBuilder envelopeBuilder = new EnvelopeBuilder(topLeft, bottomRight); +// String wkt = envelopeBuilder.toWKT(); +// s.setBoundingbox(wkt); + }); if (ace.getSpatial().size()==2) { Spatial uniqueSpatial = new Spatial(); boolean uniquePlaceNameFound = ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).count()==1; @@ -118,13 +152,13 @@ public class BulkUpload { } } - if (ace.getTemporal()!=null) { - ace.getTemporal().stream() - .filter(t->t.getMatchingPeriodOName()!=null) - .forEach(t->{ - t.setPeriodName(t.getMatchingPeriodOName()); - }); - } +// if (ace.getTemporal()!=null) { +// ace.getTemporal().stream() +// .filter(t->t.getMatchingPeriodOName()!=null) +// .forEach(t->{ +// t.setPeriodName(t.getMatchingPeriodOName()); +// }); +// } if (!isCollection) { String uniqueIsPartOf = ace.getUniqueIsPartOf(); @@ -139,15 +173,7 @@ public class BulkUpload { String[] splits = ace.getIdentifier().split("/"); -// if (ace.getAatSubjects() != null && ace.getDerivedSubject() != null) { -// String aatSource = ace.getAatSubjects().get(0).getId(); -// ace.getDerivedSubject().forEach(d -> { -// d.setSource(aatSource); -// }); -// String [] aatSourceSplit = aatSource.split("/"); -// String aatSubjectId = aatSourceSplit[aatSourceSplit.length-1]; -// ace.getAatSubjects().forEach(s -> s.setId(aatSubjectId)); -// } + log.debug("JSON >>>> "+ace.toJson()); String idES = splits[splits.length-1]; request.add(new IndexRequest(elasticSearchIndexName).id(idES) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java deleted file mode 100644 index ec8c075..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java +++ /dev/null @@ -1,46 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; - -public class AriadneGeoPoint { - private float lat; - private float lon; - - public float getLat() { - return lat; - } - - public void setLat(float lat) { - this.lat = lat; - } - - public float getLon() { - return lon; - } - - public void setLon(float lon) { - this.lon = lon; - } - - public AriadneGeoPoint() { - } - - public static AriadneGeoPoint fromJson (String json){ - return new Gson().fromJson(json, AriadneGeoPoint.class); - } - -// public static AriadneGeoPoint fromRDFJson(JsonElement json){ -// AriadneGeoPoint agp = new AriadneGeoPoint(); -// for (Map.Entry stringJsonElementEntry : json.getAsJsonObject().entrySet()) { -// switch (stringJsonElementEntry.getKey()){ -// case "https://www.ariadne-infrastructure.eu/property/lat": -// agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/lon": -// agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// } -// } -// return agp; -// } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoShape.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoShape.java deleted file mode 100644 index 34f8d4c..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoShape.java +++ /dev/null @@ -1,46 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; - -public class AriadneGeoShape { - private float lat; - private float lon; - - public float getLat() { - return lat; - } - - public void setLat(float lat) { - this.lat = lat; - } - - public float getLon() { - return lon; - } - - public void setLon(float lon) { - this.lon = lon; - } - - public AriadneGeoShape() { - } - - public static AriadneGeoShape fromJson (String json){ - return new Gson().fromJson(json, AriadneGeoShape.class); - } - -// public static AriadneGeoPoint fromRDFJson(JsonElement json){ -// AriadneGeoPoint agp = new AriadneGeoPoint(); -// for (Map.Entry stringJsonElementEntry : json.getAsJsonObject().entrySet()) { -// switch (stringJsonElementEntry.getKey()){ -// case "https://www.ariadne-infrastructure.eu/property/lat": -// agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "https://www.ariadne-infrastructure.eu/property/lon": -// agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// } -// } -// return agp; -// } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java index c09cc74..5b8b875 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java @@ -31,7 +31,7 @@ public class AriadnePlusEntry { private List spatial; private List temporal; private String title; - private NativePeriod nativePeriod; + private List nativePeriod; private String wasCreated; private DigitalImage digitalImage; @@ -268,14 +268,6 @@ public class AriadnePlusEntry { this.ariadneSubject = ariadneSubject; } - public NativePeriod getNativePeriod() { - return nativePeriod; - } - - public void setNativePeriod(NativePeriod nativePeriod) { - this.nativePeriod = nativePeriod; - } - public DigitalImage getDigitalImage() { return digitalImage; } @@ -284,6 +276,14 @@ public class AriadnePlusEntry { this.digitalImage = digitalImage; } + public List getNativePeriod() { + return nativePeriod; + } + + public void setNativePeriod(List nativePeriod) { + this.nativePeriod = nativePeriod; + } + public String toJson(){ return new Gson().toJson(this); } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java index 31a34d0..f4f93f5 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java @@ -9,8 +9,8 @@ public class Spatial { private String placeName; private String address; private GeoPoint geopoint; - private GeoShape boundingbox; - private GeoShape polygon; + private String boundingbox; + private String polygon; private String spatialPrecision; private transient String coordinatePrecision; @@ -69,22 +69,6 @@ public class Spatial { this.placeName = placeName; } - public GeoShape getBoundingbox() { - return boundingbox; - } - - public void setBoundingbox(GeoShape boundingbox) { - this.boundingbox = boundingbox; - } - - public GeoShape getPolygon() { - return polygon; - } - - public void setPolygon(GeoShape polygon) { - this.polygon = polygon; - } - public String getSpatialPrecision() { return spatialPrecision; } @@ -117,10 +101,26 @@ public class Spatial { this.lon = lon; } + public String getBoundingbox() { + return boundingbox; + } + + public void setBoundingbox(String boundingbox) { + this.boundingbox = boundingbox; + } + public GeoPoint getGeopoint() { return geopoint; } + public String getPolygon() { + return polygon; + } + + public void setPolygon(String polygon) { + this.polygon = polygon; + } + public void setGeopoint(GeoPoint geopoint) { this.geopoint = geopoint; } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Temporal.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Temporal.java index 020e894..76c5758 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Temporal.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Temporal.java @@ -10,7 +10,6 @@ public class Temporal { private String periodName; private String until; private String uri; - private transient String matchingPeriodOName; public String getFrom() { return from; @@ -50,12 +49,4 @@ public class Temporal { public static Temporal fromJson(String json){ return new Gson().fromJson(json, Temporal.class); } - - public String getMatchingPeriodOName() { - return matchingPeriodOName; - } - - public void setMatchingPeriodOName(String matchingPeriodOName) { - this.matchingPeriodOName = matchingPeriodOName; - } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql index e74a01a..d09e64a 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -47,7 +47,7 @@ CONSTRUCT { ?temporal aoprop:from ?temporalFrom . ?temporal aoprop:until ?temporalUntil . ?temporal aoprop:uri ?temporal . -%record aoprop:temporal ?temporalNative . +%record aoprop:temporalNative ?temporalNative . ?temporalNative aoprop:nativePeriodName ?temporalNativePeriodName . ?temporalNative aoprop:from ?temporalNativeFrom . ?temporalNative aoprop:until ?temporalNativeUntil . diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index 0411ea4..2dabaed 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -47,7 +47,7 @@ CONSTRUCT { ?temporal aoprop:from ?temporalFrom . ?temporal aoprop:until ?temporalUntil . ?temporal aoprop:uri ?temporal . -%record aoprop:temporal ?temporalNative . +%record aoprop:temporalNative ?temporalNative . ?temporalNative aoprop:nativePeriodName ?temporalNativePeriodName . ?temporalNative aoprop:from ?temporalNativeFrom . ?temporalNative aoprop:until ?temporalNativeUntil . diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 8835c50..7cad5bb 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -131,7 +131,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadADSArchivesBoundingBoxTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/90D1C95D-E249-3E74-92D9-B58FDF690CC7"; From 54b76c79a022d7b82154dfc12d3cd8ef8be05542 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Fri, 18 Jun 2021 19:32:12 +0200 Subject: [PATCH 07/13] added polygon support; the correct order of the geopoints describing the polygon is needed --- .../ariadneplus/elasticsearch/BulkUpload.java | 15 ++++ .../elasticsearch/model/AriadneGeoPoint.java | 32 ++++++++ .../elasticsearch/model/Spatial.java | 11 +++ .../src/main/resources/application.properties | 73 ++++++++++++------- .../sparql/read_record_data_template.sparql | 10 +++ .../GraphDbReaderAndESIndexTest.java | 9 +++ 6 files changed, 122 insertions(+), 28 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 5732aee..9aacfd7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -112,6 +112,21 @@ public class BulkUpload { // String wkt = envelopeBuilder.toWKT(); // s.setBoundingbox(wkt); }); + ace.getSpatial() + .stream() + .filter(s -> Objects.nonNull(s.getPolygonGeoPoints())&&s.getPolygonGeoPoints().size()>=4) + .forEach(s -> { + CoordinatesBuilder coordinatesBuilder = new CoordinatesBuilder(); + s.getPolygonGeoPoints().forEach(p -> { + coordinatesBuilder.coordinate( + Double.parseDouble(p.getLon()), + Double.parseDouble(p.getLat())); + }); + PolygonBuilder polygonBuilder = new PolygonBuilder(coordinatesBuilder); + String wkt = polygonBuilder.toWKT(); + log.debug("POLYGON "+wkt); + s.setPolygon(wkt); + }); if (ace.getSpatial().size()==2) { Spatial uniqueSpatial = new Spatial(); boolean uniquePlaceNameFound = ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).count()==1; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java new file mode 100644 index 0000000..944589c --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java @@ -0,0 +1,32 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class AriadneGeoPoint { + private String lat; + private String lon; + + public String getLat() { + return lat; + } + + public void setLat(String lat) { + this.lat = lat; + } + + public String getLon() { + return lon; + } + + public void setLon(String lon) { + this.lon = lon; + } + + public AriadneGeoPoint() { + + } + + public static AriadneGeoPoint fromJson(String json){ + return new Gson().fromJson(json, AriadneGeoPoint.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java index f4f93f5..0be7909 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java @@ -4,6 +4,8 @@ import com.google.gson.Gson; import org.apache.lucene.spatial3d.geom.GeoShape; import org.elasticsearch.common.geo.GeoPoint; +import java.util.List; + public class Spatial { private String placeName; @@ -20,6 +22,7 @@ public class Spatial { private transient String boundingBoxMinLon; private transient String lat; private transient String lon; + private transient List polygonGeoPoints; public String getAddress() { return address; @@ -125,6 +128,14 @@ public class Spatial { this.geopoint = geopoint; } + public List getPolygonGeoPoints() { + return polygonGeoPoints; + } + + public void setPolygonGeoPoints(List polygonGeoPoints) { + this.polygonGeoPoints = polygonGeoPoints; + } + public Spatial() { } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 23926db..7d44bd1 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -18,6 +18,21 @@ general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. type.path=https://www.ariadne-infrastructure.eu/property/rdfType exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "https://www.ariadne-infrastructure.eu/property/rdfType"] class.map.specifications={\ +"AriadneGeoPoint": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/lat": {\ +"class_field": "Lat",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lon": {\ +"class_field": "Lon",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ "Temporal": {\ "class_type": "prototype",\ "mappings": {\ @@ -26,13 +41,8 @@ class.map.specifications={\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"https://www.ariadne-infrastructure.eu/property/nativePeriodName": {\ -"class_field": "PeriodName",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ "https://www.ariadne-infrastructure.eu/property/periodOName": {\ -"class_field": "MatchingPeriodOName",\ +"class_field": "PeriodName",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ @@ -48,16 +58,21 @@ class.map.specifications={\ }\ }\ },\ -"AriadneGeoPoint": {\ -"class_type": "unique",\ +"NativePeriod": {\ +"class_type": "prototype",\ "mappings": {\ -"https://www.ariadne-infrastructure.eu/property/lat": {\ -"class_field": "Lat",\ +"https://www.ariadne-infrastructure.eu/property/from": {\ +"class_field": "From",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"https://www.ariadne-infrastructure.eu/property/lon": {\ -"class_field": "Lon",\ +"https://www.ariadne-infrastructure.eu/property/nativePeriodName": {\ +"class_field": "PeriodName",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/until": {\ +"class_field": "Until",\ "substring": "no",\ "element_type": "java.lang.String"\ }\ @@ -70,21 +85,6 @@ class.map.specifications={\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"https://www.ariadne-infrastructure.eu/property/spatialCoordinateSystem": {\ -"class_field": "CoordinateSystem",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/country": {\ -"class_field": "Country",\ -"substring": "no",\ -"element_type": "java.lang.String"\ -},\ -"https://www.ariadne-infrastructure.eu/property/location": {\ -"class_field": "Location",\ -"external_reference": "AriadneGeoPoint",\ -"substring": "no"\ -},\ "https://www.ariadne-infrastructure.eu/property/lat": {\ "class_field": "Lat",\ "substring": "no",\ @@ -119,7 +119,14 @@ class.map.specifications={\ "class_field": "BoundingBoxMinLon",\ "substring": "no",\ "element_type": "java.lang.String"\ -}}\ +},\ +"https://www.ariadne-infrastructure.eu/property/geoPoint": {\ +"class_field": "PolygonGeoPoints",\ +"substring": "no",\ +"element_type": "java.lang.String",\ +"external_reference": "AriadneGeoPoint"\ +}\ +}\ },\ "AriadneSubject": {\ "class_type": "prototype",\ @@ -269,6 +276,11 @@ class.map.specifications={\ "substring": "no",\ "external_reference": "Spatial"\ },\ +"https://www.ariadne-infrastructure.eu/property/spatialRegionPolygon": {\ +"class_field": "Spatial",\ +"substring": "no",\ +"external_reference": "Spatial"\ +},\ "https://www.ariadne-infrastructure.eu/property/landingPage": {\ "class_field": "LandingPage",\ "substring": "no",\ @@ -319,6 +331,11 @@ class.map.specifications={\ "substring": "no",\ "external_reference": "Temporal"\ },\ +"https://www.ariadne-infrastructure.eu/property/temporalNative": {\ +"class_field": "NativePeriod",\ +"substring": "no",\ +"external_reference": "NativePeriod"\ +},\ "https://www.ariadne-infrastructure.eu/property/language": {\ "class_field": "Language",\ "substring": "no",\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index 2dabaed..aec0535 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -42,6 +42,10 @@ CONSTRUCT { ?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon . ?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat . ?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon . +%record aoprop:spatialRegionPolygon ?spatialRegionPolygon . +?spatialRegionPolygon aoprop:geoPoint ?geoPoint . +?geoPoint aoprop:lat ?lat . +?geoPoint aoprop:lon ?lon . %record aoprop:temporal ?temporal . ?temporal aoprop:periodOName ?temporalPeriodName . ?temporal aoprop:from ?temporalFrom . @@ -177,6 +181,12 @@ where { aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ; aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ; } + optional { + %record aocat:has_spatial_coverage ?spatialRegionPolygon . + ?spatialRegionPolygon aocat:has_polygonal_representation ?geoPoint . + ?geoPoint aocat:has_latitude ?lat . + ?geoPoint aocat:has_longitude ?lon . + } } } } diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 7cad5bb..4d1e98f 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -160,6 +160,15 @@ public class GraphDbReaderAndESIndexTest { readAndIndexTest(isRecord, recordId, datasource, collectionId); } + @Test + public void uploadDansPolygonTest() throws Exception { + boolean isRecord = true; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B"; + String datasource = "dans"; + String collectionId = "easy"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + @Test @Ignore public void uploadSndRockartTest() throws Exception { From 41896d312be82e26952b237bcfd6ca74c45c59f7 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Mon, 21 Jun 2021 15:20:39 +0200 Subject: [PATCH 08/13] fix for from and until values on temporal and native period fields --- .../ariadneplus/elasticsearch/BulkUpload.java | 28 +++++------ .../src/main/resources/application.properties | 4 +- .../read_collection_data_template.sparql | 21 ++++---- .../sparql/read_record_data_template.sparql | 8 ++-- .../GraphDbReaderAndESIndexTest.java | 48 +++++++++++-------- 5 files changed, 57 insertions(+), 52 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 9aacfd7..4c2bb1d 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -1,15 +1,12 @@ package eu.dnetlib.ariadneplus.elasticsearch; -import eu.dnetlib.ariadneplus.elasticsearch.model.*; +import eu.dnetlib.ariadneplus.elasticsearch.model.AriadnePlusEntry; +import eu.dnetlib.ariadneplus.elasticsearch.model.Spatial; import eu.dnetlib.ariadneplus.reader.ResourceManager; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpHost; -import org.apache.lucene.spatial3d.geom.GeoPoint; -import org.apache.lucene.spatial3d.geom.GeoPolygon; -import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; -import org.apache.lucene.spatial3d.geom.PlanetModel; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; @@ -17,18 +14,17 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.geo.builders.CoordinatesBuilder; -import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.geometry.Polygon; -import org.locationtech.jts.geom.Coordinate; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import java.io.IOException; -import java.io.OutputStream; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; @Service @@ -70,9 +66,9 @@ public class BulkUpload { AriadnePlusEntry ace = ((AriadnePlusEntry) next); if (isCollection) { ace.setResourceType("collection"); - if (ace.getSpatial()==null) { - ace.setSpatial(Arrays.asList(new Spatial())); - } +// if (ace.getSpatial()==null) { +// ace.setSpatial(Arrays.asList(new Spatial())); +// } } else { ace.setResourceType("dataset"); @@ -162,9 +158,9 @@ public class BulkUpload { // ace.getSpatial().removeIf(s -> (s.getPlaceName()!=null&&s.getPlaceName().equals("Name not provided")&&Objects.isNull(s.getLocation()))); // } } - else { - ace.setSpatial(Arrays.asList(new Spatial())); - } +// else { +// ace.setSpatial(Arrays.asList(new Spatial())); +// } } // if (ace.getTemporal()!=null) { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 7d44bd1..febb6f7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -61,7 +61,7 @@ class.map.specifications={\ "NativePeriod": {\ "class_type": "prototype",\ "mappings": {\ -"https://www.ariadne-infrastructure.eu/property/from": {\ +"https://www.ariadne-infrastructure.eu/property/nativeFrom": {\ "class_field": "From",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -71,7 +71,7 @@ class.map.specifications={\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"https://www.ariadne-infrastructure.eu/property/until": {\ +"https://www.ariadne-infrastructure.eu/property/nativeUntil": {\ "class_field": "Until",\ "substring": "no",\ "element_type": "java.lang.String"\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql index d09e64a..0597db8 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -49,8 +49,8 @@ CONSTRUCT { ?temporal aoprop:uri ?temporal . %record aoprop:temporalNative ?temporalNative . ?temporalNative aoprop:nativePeriodName ?temporalNativePeriodName . -?temporalNative aoprop:from ?temporalNativeFrom . -?temporalNative aoprop:until ?temporalNativeUntil . +?temporalNative aoprop:nativeFrom ?temporalNativeFrom . +?temporalNative aoprop:nativeUntil ?temporalNativeUntil . %record aoprop:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . %record aoprop:resourceType ?resourceType . @@ -113,19 +113,22 @@ where { graph { %record aocat:has_temporal_coverage ?temporalNative . ?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName . - } - graph { optional { ?temporalNative aocat:from ?temporalNativeFrom . ?temporalNative aocat:until ?temporalNativeUntil . } } } - graph { - optional { - %record aocat:has_temporal_coverage ?temporal . + } + } + + optional { + graph { + %record aocat:has_temporal_coverage ?temporal . + } + graph { ?temporal aocat:has_period ?label . - graph { + graph { ?label skos:prefLabel ?temporalPeriodName . } optional { @@ -134,8 +137,6 @@ where { } } } - } - } graph { { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index aec0535..e2de085 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -53,8 +53,8 @@ CONSTRUCT { ?temporal aoprop:uri ?temporal . %record aoprop:temporalNative ?temporalNative . ?temporalNative aoprop:nativePeriodName ?temporalNativePeriodName . -?temporalNative aoprop:from ?temporalNativeFrom . -?temporalNative aoprop:until ?temporalNativeUntil . +?temporalNative aoprop:nativeFrom ?temporalNativeFrom . +?temporalNative aoprop:nativeUntil ?temporalNativeUntil . %record aoprop:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . %record aoprop:resourceType ?resourceType . @@ -100,9 +100,7 @@ where { %record aocat:has_temporal_coverage ?temporalNative . ?temporalNative aocat:has_native_period ?nativeLabel . ?nativeLabel skos:prefLabel ?temporalNativePeriodName . - } - graph { - optional { + optional { ?temporalNative aocat:from ?temporalNativeFrom . ?temporalNative aocat:until ?temporalNativeUntil . } diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 4d1e98f..1fdc621 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -32,7 +32,7 @@ public class GraphDbReaderAndESIndexTest { private RunSPARQLQueryService runSPQRLQuery; @Test - @Ignore +// @Ignore public void uploadAMCRFieldworkTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/E61E0F4E-268F-39E4-8EDB-A431AFC505AA"; @@ -42,7 +42,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadAMCRDocumentTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/AMCR/FC59581D-DC3A-31DA-922A-98DE764F3D76"; @@ -52,7 +52,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadAMCRSiteTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/3C7EC936-A7CA-3720-B3DC-413A25754FD4"; @@ -62,7 +62,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadAMCRIndividualFindTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/AMCR/98D717C4-410F-35C6-8072-FABA7686B4A3"; @@ -81,7 +81,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadADSRecordWithNativeFromUntilTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/3C3C7A86-FF09-3431-95B1-B9A4AA8293AF"; @@ -91,7 +91,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadADSRecordWithoutNativeFromUntilTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DF5F27D3-C877-3F23-9EAA-3776362363AA"; @@ -101,7 +101,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadZbivaRecordSpatialTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/ZRC-SAZU-Zbiva/B34517C6-8D94-3A02-B461-08522F958479"; @@ -111,7 +111,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadHNMCollectionSpatialTest() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/5A7A4257-EE73-31F9-9F74-BADB371555F5"; @@ -121,10 +121,10 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadHNMCollectionTemporalTest() throws Exception { boolean isRecord = false; - String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/0F1AF07E-CE63-342C-8E95-11FF1C024BD5"; + String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/HNM/1AE50143-45C7-304F-8367-BCF3606CEF10"; String datasource = "hnm"; String collectionId = "hnmad"; readAndIndexTest(isRecord, recordId, datasource, collectionId); @@ -141,8 +141,8 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore - public void uploadDansNoSpatialTest() throws Exception { +// @Ignore + public void uploadDansSpatialTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/F100A0AD-6A7F-3976-B77F-FFAB4F5B55DD"; String datasource = "dans"; @@ -151,8 +151,8 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore - public void uploadDansFieldworkReportTest() throws Exception { +// @Ignore + public void uploadDansNativePeriodTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/CA076E46-5CED-322C-B77E-3B90C11B968B"; String datasource = "dans"; @@ -161,7 +161,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - public void uploadDansPolygonTest() throws Exception { + public void uploadDansTemporalPolygonTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B"; String datasource = "dans"; @@ -170,7 +170,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadSndRockartTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/SHFA/FED23426-2C68-3BB3-9BBA-24F2077C9C6A"; @@ -180,7 +180,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadNIAMTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/NIAM-BAS%2FAKB/D4388BF5-AF3D-3F24-8C2E-2FEAD1255FB1"; @@ -190,7 +190,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadADS398Test() throws Exception { boolean isRecord = false; String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/AAA81A6D-56F3-341C-BAF0-791C31BC7F73"; @@ -200,7 +200,7 @@ public class GraphDbReaderAndESIndexTest { } @Test - @Ignore +// @Ignore public void uploadROADTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FC70B370-C489-31C5-B1D4-339CFD28CF2B"; @@ -209,6 +209,16 @@ public class GraphDbReaderAndESIndexTest { readAndIndexTest(isRecord, recordId, datasource, collectionId); } + @Test +// @Ignore + public void uploadADS328NativeFromUntilTest() throws Exception { + boolean isRecord = true; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/7516E8FC-7C1E-38F7-8D62-8BF96B0D2559"; + String datasource = "ads"; + String collectionId = "328"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties(); From 89f309feaa6d785340b5aceed62811fb19b0551c Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 22 Jun 2021 11:06:56 +0200 Subject: [PATCH 09/13] added mapping for polygon data when aocat:has_polygonal_representation is in wkt format (snd::zip) --- .../ariadneplus/elasticsearch/BulkUpload.java | 14 ++++++++++++ .../elasticsearch/model/Spatial.java | 9 ++++++++ .../src/main/resources/application.properties | 10 +++++++++ .../read_collection_data_template.sparql | 14 +++++++----- .../sparql/read_record_data_template.sparql | 14 +++++++----- .../GraphDbReaderAndESIndexTest.java | 22 +++++++++++++++++++ 6 files changed, 73 insertions(+), 10 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 4c2bb1d..eed24d7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -66,6 +66,14 @@ public class BulkUpload { AriadnePlusEntry ace = ((AriadnePlusEntry) next); if (isCollection) { ace.setResourceType("collection"); + if (ace.getSpatial()!=null) { + ace.getSpatial() + .stream() + .filter(s -> Objects.nonNull(s.getWkt())) + .forEach(s -> { + s.setPolygon(s.getWkt()); + }); + } // if (ace.getSpatial()==null) { // ace.setSpatial(Arrays.asList(new Spatial())); // } @@ -123,6 +131,12 @@ public class BulkUpload { log.debug("POLYGON "+wkt); s.setPolygon(wkt); }); + ace.getSpatial() + .stream() + .filter(s -> Objects.nonNull(s.getWkt())) + .forEach(s -> { + s.setPolygon(s.getWkt()); + }); if (ace.getSpatial().size()==2) { Spatial uniqueSpatial = new Spatial(); boolean uniquePlaceNameFound = ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).count()==1; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java index 0be7909..efd7220 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java @@ -23,6 +23,7 @@ public class Spatial { private transient String lat; private transient String lon; private transient List polygonGeoPoints; + private transient String wkt; public String getAddress() { return address; @@ -136,6 +137,14 @@ public class Spatial { this.polygonGeoPoints = polygonGeoPoints; } + public String getWkt() { + return wkt; + } + + public void setWkt(String wkt) { + this.wkt = wkt; + } + public Spatial() { } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index febb6f7..6e87911 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -125,6 +125,11 @@ class.map.specifications={\ "substring": "no",\ "element_type": "java.lang.String",\ "external_reference": "AriadneGeoPoint"\ +},\ +"https://www.ariadne-infrastructure.eu/property/wkt": {\ +"class_field": "Wkt",\ +"substring": "no",\ +"element_type": "java.lang.String"\ }\ }\ },\ @@ -281,6 +286,11 @@ class.map.specifications={\ "substring": "no",\ "external_reference": "Spatial"\ },\ +"https://www.ariadne-infrastructure.eu/property/spatialRegionWKTPolygon": {\ +"class_field": "Spatial",\ +"substring": "no",\ +"external_reference": "Spatial"\ +},\ "https://www.ariadne-infrastructure.eu/property/landingPage": {\ "class_field": "LandingPage",\ "substring": "no",\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql index 0597db8..d40b53d 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -33,7 +33,6 @@ CONSTRUCT { %record aoprop:landingPage ?landingPage . %record aoprop:spatialRegion ?spatialRegion . ?spatialRegion aoprop:placeName ?spatialPlaceName . -?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . %record aoprop:spatialRegionPoint ?spatialRegionPoint . ?spatialRegionPoint aoprop:lat ?spatialLocationLat . ?spatialRegionPoint aoprop:lon ?spatialLocationLon . @@ -42,6 +41,8 @@ CONSTRUCT { ?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon . ?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat . ?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon . +%record aoprop:spatialRegionWKTPolygon ?spatialRegionWKTPolygon . +?spatialRegionWKTPolygon aoprop:wkt ?wkt . %record aoprop:temporal ?temporal . ?temporal aoprop:periodOName ?temporalPeriodName . ?temporal aoprop:from ?temporalFrom . @@ -179,10 +180,6 @@ where { %record aocat:has_spatial_coverage ?spatialRegion . ?spatialRegion aocat:has_place_name ?spatialPlaceName . } - optional { - %record aocat:has_spatial_coverage ?spatialRegion . - ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem . - } optional { %record aocat:has_spatial_coverage ?spatialRegionPoint . ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; @@ -195,6 +192,13 @@ where { aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ; aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ; } + optional { + %record aocat:has_spatial_coverage ?spatialRegionWKTPolygon . + ?spatialRegionWKTPolygon rdf:type aocat:AO_Spatial_Region_Polygon . + ?spatialRegionWKTPolygon aocat:has_polygonal_representation ?wkt . + filter (datatype(?wkt) = rdf:XMLLiteral) + filter (!isIRI(?wkt)) + } } } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index e2de085..86dc7ae 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -33,7 +33,6 @@ CONSTRUCT { %record aoprop:landingPage ?landingPage . %record aoprop:spatialRegion ?spatialRegion . ?spatialRegion aoprop:placeName ?spatialPlaceName . -?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . %record aoprop:spatialRegionPoint ?spatialRegionPoint . ?spatialRegionPoint aoprop:lat ?spatialLocationLat . ?spatialRegionPoint aoprop:lon ?spatialLocationLon . @@ -46,6 +45,8 @@ CONSTRUCT { ?spatialRegionPolygon aoprop:geoPoint ?geoPoint . ?geoPoint aoprop:lat ?lat . ?geoPoint aoprop:lon ?lon . +%record aoprop:spatialRegionWKTPolygon ?spatialRegionWKTPolygon . +?spatialRegionWKTPolygon aoprop:wkt ?wkt . %record aoprop:temporal ?temporal . ?temporal aoprop:periodOName ?temporalPeriodName . ?temporal aoprop:from ?temporalFrom . @@ -163,10 +164,6 @@ where { where { %record aocat:has_spatial_coverage ?spatialRegion . ?spatialRegion aocat:has_place_name ?spatialPlaceName . - optional { - %record aocat:has_spatial_coverage ?spatialRegion . - ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem . - } optional { %record aocat:has_spatial_coverage ?spatialRegionPoint . ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; @@ -185,6 +182,13 @@ where { ?geoPoint aocat:has_latitude ?lat . ?geoPoint aocat:has_longitude ?lon . } + optional { + %record aocat:has_spatial_coverage ?spatialRegionWKTPolygon . + ?spatialRegionWKTPolygon rdf:type aocat:AO_Spatial_Region_Polygon . + ?spatialRegionWKTPolygon aocat:has_polygonal_representation ?wkt . + filter (datatype(?wkt) = rdf:XMLLiteral) + filter (!isIRI(?wkt)) + } } } } diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 1fdc621..d721238 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -72,6 +72,7 @@ public class GraphDbReaderAndESIndexTest { } @Test +// @Ignore public void uploadADSRecordTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/34E3811A-0BAD-3832-B3A0-3139E8A0285C"; @@ -161,6 +162,7 @@ public class GraphDbReaderAndESIndexTest { } @Test +// @Ignore public void uploadDansTemporalPolygonTest() throws Exception { boolean isRecord = true; String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/D4E12349-E214-3F3F-BEE4-D39D9138916B"; @@ -219,6 +221,26 @@ public class GraphDbReaderAndESIndexTest { readAndIndexTest(isRecord, recordId, datasource, collectionId); } + @Test +// @Ignore + public void uploadFastiMultipleAriadneSubjectTest() throws Exception { + boolean isRecord = true; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/FASTIONLINE/FE740BC7-2D9E-3644-A88E-3E8E5A5EA95F"; + String datasource = "fasti"; + String collectionId = "fieldwork"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + + @Test +// @Ignore + public void uploadSNDArchivePolygonTest() throws Exception { + boolean isRecord = false; + String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/A7A2588A-1FE7-3FDD-AD66-C84EACE1860E"; + String datasource = "snd"; + String collectionId = "zip"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties(); From 4882e38d641136d7d53bd06a551e9ef47532e4cb Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Wed, 23 Jun 2021 11:26:56 +0200 Subject: [PATCH 10/13] added new fields from es mapping: agent (homepage,institution,agentIdentifier) and wasCreated --- .../ariadneplus/elasticsearch/BulkUpload.java | 8 ++-- .../elasticsearch/model/AriadnePlusEntry.java | 2 +- .../src/main/resources/application.properties | 10 +++++ .../read_collection_data_template.sparql | 43 +++++++++++++++++++ .../sparql/read_record_data_template.sparql | 43 +++++++++++++++++++ 5 files changed, 101 insertions(+), 5 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index eed24d7..aae28c5 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -126,10 +126,10 @@ public class BulkUpload { Double.parseDouble(p.getLon()), Double.parseDouble(p.getLat())); }); - PolygonBuilder polygonBuilder = new PolygonBuilder(coordinatesBuilder); - String wkt = polygonBuilder.toWKT(); - log.debug("POLYGON "+wkt); - s.setPolygon(wkt); +// PolygonBuilder polygonBuilder = new PolygonBuilder(coordinatesBuilder); +// String wkt = polygonBuilder.toWKT(); +// log.debug("POLYGON "+wkt); +// s.setPolygon(wkt); }); ace.getSpatial() .stream() diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java index 5b8b875..d3be760 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java @@ -225,7 +225,7 @@ public class AriadnePlusEntry { } public void setWasCreated(String wasCreated) { - this.wasCreated = wasCreated; + this.wasCreated = ESUtils.getESFormatDate(wasCreated); } public String getUniqueIsPartOf() { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 6e87911..1f92354 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -170,6 +170,11 @@ class.map.specifications={\ "class_field": "Institution",\ "substring": "no",\ "element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/agentIdentifier": {\ +"class_field": "AgentIdentifier",\ +"substring": "no",\ +"element_type": "java.lang.String"\ }\ }\ },\ @@ -246,6 +251,11 @@ class.map.specifications={\ "element_type": "java.lang.String",\ "substring": "no"\ },\ +"https://www.ariadne-infrastructure.eu/property/created": {\ +"class_field": "WasCreated",\ +"element_type": "java.lang.String",\ +"substring": "no"\ +},\ "https://www.ariadne-infrastructure.eu/property/resourceType": {\ "class_field": "ResourceType",\ "element_type": "java.lang.String",\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql index d40b53d..1165ca3 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -12,22 +12,38 @@ CONSTRUCT { %record aoprop:originalId ?originalId . %record aoprop:issued ?issued . %record aoprop:modified ?modified . +%record aoprop:created ?created . %record aoprop:partOf ?partOf . %record aoprop:creator ?creator . ?creator aoprop:name ?creatorName . ?creator aoprop:email ?creatorEmail . +?creator aoprop:homepage ?creatorHomepage . +?creator aoprop:institution ?creatorInstitution . +?creator aoprop:agentIdentifier ?creator . %record aoprop:contributor ?contributor . ?contributor aoprop:name ?contributorName . ?contributor aoprop:email ?contributorEmail . +?contributor aoprop:homepage ?contributorHomepage . +?contributor aoprop:institution ?contributorInstitution . +?contributor aoprop:agentIdentifier ?contributor . %record aoprop:legalResponsible ?legalResponsible . ?legalResponsible aoprop:name ?legalResponsibleName . ?legalResponsible aoprop:email ?legalResponsibleEmail . +?legalResponsible aoprop:homepage ?legalResponsibleHomepage . +?legalResponsible aoprop:institution ?legalResponsibleInstitution . +?legalResponsible aoprop:agentIdentifier ?legalResponsible . %record aoprop:owner ?owner . ?owner aoprop:name ?ownerName . ?owner aoprop:email ?ownerEmail . +?owner aoprop:homepage ?ownerHomepage . +?owner aoprop:institution ?ownerInstitution . +?owner aoprop:agentIdentifier ?owner . %record aoprop:publisher ?publisher . ?publisher aoprop:name ?publisherName . ?publisher aoprop:email ?publisherEmail . +?publisher aoprop:homepage ?publisherHomepage . +?publisher aoprop:institution ?publisherInstitution . +?publisher aoprop:agentIdentifier ?publisher . %record aoprop:accessPolicy ?accessPolicy . %record aoprop:accessRights ?accessRights . %record aoprop:landingPage ?landingPage . @@ -156,18 +172,45 @@ where { %record aocat:has_access_rights ?accessRights . %record aocat:has_ARIADNE_subject ?archeologicalResourceType . ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + %record aocat:was_created_on ?created . + } optional { ?contributor aocat:has_email ?contributorEmail . } + optional { + ?contributor aocat:has_homepage ?contributorHomepage . + } + optional { + ?contributor aocat:has_institution ?contributorInstitution . + } optional { ?legalResponsible aocat:has_email ?legalResponsibleEmail . } + optional { + ?legalResponsible aocat:has_homepage ?legalResponsibleHomepage . + } + optional { + ?legalResponsible aocat:has_institution ?legalResponsibleInstitution . + } optional { ?owner aocat:has_email ?ownerEmail . } + optional { + ?owner aocat:has_homepage ?ownerHomepage . + } + optional { + ?owner aocat:has_institution ?ownerInstitution . + } optional { ?publisher aocat:has_email ?publisherEmail . } + optional { + ?publisher aocat:has_homepage ?publisherHomepage . + } + optional { + ?publisher aocat:has_institution ?publisherInstitution . + } } } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index 86dc7ae..7ff7eab 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -12,22 +12,38 @@ CONSTRUCT { %record aoprop:originalId ?originalId . %record aoprop:issued ?issued . %record aoprop:modified ?modified . +%record aoprop:created ?created . %record aoprop:partOf ?partOf . %record aoprop:creator ?creator . ?creator aoprop:name ?creatorName . ?creator aoprop:email ?creatorEmail . +?creator aoprop:homepage ?creatorHomepage . +?creator aoprop:institution ?creatorInstitution . +?creator aoprop:agentIdentifier ?creator . %record aoprop:contributor ?contributor . ?contributor aoprop:name ?contributorName . ?contributor aoprop:email ?contributorEmail . +?contributor aoprop:homepage ?contributorHomepage . +?contributor aoprop:institution ?contributorInstitution . +?contributor aoprop:agentIdentifier ?contributor . %record aoprop:legalResponsible ?legalResponsible . ?legalResponsible aoprop:name ?legalResponsibleName . ?legalResponsible aoprop:email ?legalResponsibleEmail . +?legalResponsible aoprop:homepage ?legalResponsibleHomepage . +?legalResponsible aoprop:institution ?legalResponsibleInstitution . +?legalResponsible aoprop:agentIdentifier ?legalResponsible . %record aoprop:owner ?owner . ?owner aoprop:name ?ownerName . ?owner aoprop:email ?ownerEmail . +?owner aoprop:homepage ?ownerHomepage . +?owner aoprop:institution ?ownerInstitution . +?owner aoprop:agentIdentifier ?owner . %record aoprop:publisher ?publisher . ?publisher aoprop:name ?publisherName . ?publisher aoprop:email ?publisherEmail . +?publisher aoprop:homepage ?publisherHomepage . +?publisher aoprop:institution ?publisherInstitution . +?publisher aoprop:agentIdentifier ?publisher . %record aoprop:accessPolicy ?accessPolicy . %record aoprop:accessRights ?accessRights . %record aoprop:landingPage ?landingPage . @@ -142,18 +158,45 @@ where { %record aocat:has_access_rights ?accessRights . %record aocat:has_ARIADNE_subject ?archeologicalResourceType . ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + %record aocat:was_created_on ?created . + } optional { ?contributor aocat:has_email ?contributorEmail . } + optional { + ?contributor aocat:has_homepage ?contributorHomepage . + } + optional { + ?contributor aocat:has_institution ?contributorInstitution . + } optional { ?legalResponsible aocat:has_email ?legalResponsibleEmail . } + optional { + ?legalResponsible aocat:has_homepage ?legalResponsibleHomepage . + } + optional { + ?legalResponsible aocat:has_institution ?legalResponsibleInstitution . + } optional { ?owner aocat:has_email ?ownerEmail . } + optional { + ?owner aocat:has_homepage ?ownerHomepage . + } + optional { + ?owner aocat:has_institution ?ownerInstitution . + } optional { ?publisher aocat:has_email ?publisherEmail . } + optional { + ?publisher aocat:has_homepage ?publisherHomepage . + } + optional { + ?publisher aocat:has_institution ?publisherInstitution . + } } } } From 6c8c9fcba327f21d1ec3a601bf5b712adbc8ca08 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Wed, 23 Jun 2021 14:56:25 +0200 Subject: [PATCH 11/13] added fields has_type and is_about --- .../ariadneplus/elasticsearch/BulkUpload.java | 6 +++ .../elasticsearch/model/AriadnePlusEntry.java | 42 +++++++++++++------ .../{AriadneURI.java => AriadneResource.java} | 8 ++-- .../src/main/resources/application.properties | 33 ++++++++++++++- .../read_collection_data_template.sparql | 11 +++++ .../sparql/read_record_data_template.sparql | 13 +++++- .../GraphDbReaderAndESIndexTest.java | 10 +++++ 7 files changed, 105 insertions(+), 18 deletions(-) rename dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/{AriadneURI.java => AriadneResource.java} (68%) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index aae28c5..06b3e45 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -1,6 +1,7 @@ package eu.dnetlib.ariadneplus.elasticsearch; import eu.dnetlib.ariadneplus.elasticsearch.model.AriadnePlusEntry; +import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneResource; import eu.dnetlib.ariadneplus.elasticsearch.model.Spatial; import eu.dnetlib.ariadneplus.reader.ResourceManager; import org.apache.commons.lang3.StringUtils; @@ -64,6 +65,11 @@ public class BulkUpload { try { Object next = manager.next(); AriadnePlusEntry ace = ((AriadnePlusEntry) next); + AriadneResource uri = new AriadneResource(); + uri.setUri(ace.getTypeURI()); + uri.setLabel(ace.getTypeLabel()); + ace.setHas_type(uri); + if (isCollection) { ace.setResourceType("collection"); if (ace.getSpatial()!=null) { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java index d3be760..00acbc0 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java @@ -24,9 +24,9 @@ public class AriadnePlusEntry { private String originalId; private List owner; private List publisher; - private AriadneURI is_about; + private List is_about; private String resourceType; - private AriadneURI has_type; + private AriadneResource has_type; private List responsible; private List spatial; private List temporal; @@ -36,6 +36,8 @@ public class AriadnePlusEntry { private DigitalImage digitalImage; private transient String uniqueIsPartOf; + private transient String typeURI; + private transient String typeLabel; public List getContributor() { return contributor; @@ -212,11 +214,7 @@ public class AriadnePlusEntry { this.description = description; } - public void setIs_about(AriadneURI is_about) { - this.is_about = is_about; - } - - public void setHas_type(AriadneURI has_type) { + public void setHas_type(AriadneResource has_type) { this.has_type = has_type; } @@ -244,11 +242,7 @@ public class AriadnePlusEntry { return description; } - public AriadneURI getIs_about() { - return is_about; - } - - public AriadneURI getHas_type() { + public AriadneResource getHas_type() { return has_type; } @@ -284,6 +278,30 @@ public class AriadnePlusEntry { this.nativePeriod = nativePeriod; } + public String getTypeURI() { + return typeURI; + } + + public void setTypeURI(String typeURI) { + this.typeURI = typeURI; + } + + public String getTypeLabel() { + return typeLabel; + } + + public void setTypeLabel(String typeLabel) { + this.typeLabel = typeLabel; + } + + public List getIs_about() { + return is_about; + } + + public void setIs_about(List is_about) { + this.is_about = is_about; + } + public String toJson(){ return new Gson().toJson(this); } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneURI.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneResource.java similarity index 68% rename from dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneURI.java rename to dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneResource.java index c5b67a3..7768efe 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneURI.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneResource.java @@ -2,7 +2,7 @@ package eu.dnetlib.ariadneplus.elasticsearch.model; import com.google.gson.Gson; -public class AriadneURI { +public class AriadneResource { private String label; private String uri; @@ -22,10 +22,10 @@ public class AriadneURI { this.uri = uri; } - public AriadneURI() { + public AriadneResource() { } - public static AriadneURI fromJson(String json){ - return new Gson().fromJson(json, AriadneURI.class); + public static AriadneResource fromJson(String json){ + return new Gson().fromJson(json, AriadneResource.class); } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 1f92354..7927ec6 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -18,6 +18,21 @@ general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. type.path=https://www.ariadne-infrastructure.eu/property/rdfType exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "https://www.ariadne-infrastructure.eu/property/rdfType"] class.map.specifications={\ +"AriadneResource": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/aboutURILabel": {\ +"class_field": "Label",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/aboutURI": {\ +"class_field": "Uri",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ "AriadneGeoPoint": {\ "class_type": "prototype",\ "mappings": {\ @@ -226,6 +241,16 @@ class.map.specifications={\ "AriadnePlusEntry": {\ "class_type": "unique",\ "mappings": {\ +"https://www.ariadne-infrastructure.eu/property/typeURI": {\ +"class_field": "TypeURI",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/typeLabel": {\ +"class_field": "TypeLabel",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ "https://www.ariadne-infrastructure.eu/property/accessPolicy": {\ "class_field": "AccessPolicy",\ "substring": "no",\ @@ -365,7 +390,13 @@ class.map.specifications={\ "class_field": "UniqueIsPartOf",\ "substring": "no",\ "element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/about": {\ +"class_field": "Is_about",\ +"substring": "no",\ +"external_reference": "AriadneResource"\ }\ }\ }\ -} \ No newline at end of file +} + diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql index 1165ca3..8953445 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -71,6 +71,11 @@ CONSTRUCT { %record aoprop:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . %record aoprop:resourceType ?resourceType . +%record aoprop:typeURI ?typeURI . +%record aoprop:typeLabel ?typeLabel . +%record aoprop:about ?aboutURI . +?aboutURI aoprop:aboutURI ?aboutURI . +?aboutURI aoprop:aboutURILabel ?aboutLabel . %record aoprop:nativeSubject ?nativeSubject . ?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . ?nativeSubject aoprop:rdfAbout ?nativeSubject . @@ -96,6 +101,12 @@ where { ?creator aocat:has_name ?creatorName . %record aocat:has_title ?title . %record aocat:has_type / skos:prefLabel ?resourceType . + %record aocat:has_type ?typeURI . + ?typeURI skos:prefLabel ?typeLabel . + optional { + %record aocat:is_about ?aboutURI . + ?aboutURI rdfs:label ?aboutLabel . + } optional { %record aocat:has_native_subject ?nativeSubject . ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index 7ff7eab..1a3d83a 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -75,6 +75,11 @@ CONSTRUCT { %record aoprop:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . %record aoprop:resourceType ?resourceType . +%record aoprop:typeURI ?typeURI . +%record aoprop:typeLabel ?typeLabel . +%record aoprop:about ?aboutURI . +?aboutURI aoprop:aboutURI ?aboutURI . +?aboutURI aoprop:aboutURILabel ?aboutLabel . %record aoprop:nativeSubject ?nativeSubject . ?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . ?nativeSubject aoprop:rdfAbout ?nativeSubject . @@ -97,7 +102,13 @@ where { %record aocat:has_creator ?creator . ?creator aocat:has_name ?creatorName . %record aocat:has_title ?title . - %record aocat:has_type / skos:prefLabel ?resourceType . + %record aocat:has_type / skos:prefLabel ?resourceType . + %record aocat:has_type ?typeURI . + ?typeURI skos:prefLabel ?typeLabel . + optional { + %record aocat:is_about ?aboutURI . + ?aboutURI rdfs:label ?aboutLabel . + } optional { ?creator aocat:has_email ?creatorEmail . } diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index d721238..1e29511 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -241,6 +241,16 @@ public class GraphDbReaderAndESIndexTest { readAndIndexTest(isRecord, recordId, datasource, collectionId); } + @Test +// @Ignore + public void uploadADSIsAboutTest() throws Exception { + boolean isRecord = true; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DCDF395A-2CC8-3FC2-B9A5-5A924090DF10"; + String datasource = "ads"; + String collectionId = "3"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties(); From 8944977a17b168f976289ad9aed2867c7d69cb11 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Mon, 28 Jun 2021 12:50:39 +0200 Subject: [PATCH 12/13] added digitalImage support --- .../ariadneplus/elasticsearch/BulkUpload.java | 8 ++++++++ .../elasticsearch/model/AriadnePlusEntry.java | 18 ++++++++--------- .../src/main/resources/application.properties | 20 +++++++++++++++++++ .../sparql/read_record_data_template.sparql | 9 +++++++++ .../GraphDbReaderAndESIndexTest.java | 8 ++++---- 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 06b3e45..74fbb0e 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -86,6 +86,14 @@ public class BulkUpload { } else { ace.setResourceType("dataset"); + if (ace.getDigitalImage()!=null) { + ace.getDigitalImage() + .stream() + .filter(i -> i.getProviderUri()!=null) + .forEach(i -> { + i.setPrimary("true"); + }); + } if (ace.getSpatial()!=null) { ace.getSpatial() .stream() diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java index 00acbc0..bcc8fba 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadnePlusEntry.java @@ -33,7 +33,7 @@ public class AriadnePlusEntry { private String title; private List nativePeriod; private String wasCreated; - private DigitalImage digitalImage; + private List digitalImage; private transient String uniqueIsPartOf; private transient String typeURI; @@ -262,14 +262,6 @@ public class AriadnePlusEntry { this.ariadneSubject = ariadneSubject; } - public DigitalImage getDigitalImage() { - return digitalImage; - } - - public void setDigitalImage(DigitalImage digitalImage) { - this.digitalImage = digitalImage; - } - public List getNativePeriod() { return nativePeriod; } @@ -302,6 +294,14 @@ public class AriadnePlusEntry { this.is_about = is_about; } + public List getDigitalImage() { + return digitalImage; + } + + public void setDigitalImage(List digitalImage) { + this.digitalImage = digitalImage; + } + public String toJson(){ return new Gson().toJson(this); } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 7927ec6..9c3a5b9 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -18,6 +18,21 @@ general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. type.path=https://www.ariadne-infrastructure.eu/property/rdfType exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "https://www.ariadne-infrastructure.eu/property/rdfType"] class.map.specifications={\ +"DigitalImage": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/primaryVisualComponent": {\ +"class_field": "ProviderUri",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/visualComponent": {\ +"class_field": "AriadneUri",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ "AriadneResource": {\ "class_type": "prototype",\ "mappings": {\ @@ -395,6 +410,11 @@ class.map.specifications={\ "class_field": "Is_about",\ "substring": "no",\ "external_reference": "AriadneResource"\ +},\ +"https://www.ariadne-infrastructure.eu/property/image": {\ +"class_field": "DigitalImage",\ +"substring": "no",\ +"external_reference": "DigitalImage"\ }\ }\ }\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index 1a3d83a..58704dc 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -93,6 +93,9 @@ CONSTRUCT { %record aoprop:title ?title . %record aoprop:description ?description . %record aoprop:language ?language . +%record aoprop:image ?primaryVisualComponent . +?primaryVisualComponent aoprop:primaryVisualComponent ?primaryVisualComponent . +?primaryVisualComponent aoprop:visualComponent ?visualComponent . } where { graph { @@ -121,6 +124,12 @@ where { optional { %record aocat:has_landing_page / rdfs:label ?landingPage . } + optional { + %record aocat:has_primary_visual_component ?primaryVisualComponent . + } + optional { + %record aocat:has_visual_component ?visualComponent . + } } optional { diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 1e29511..66cb721 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -243,11 +243,11 @@ public class GraphDbReaderAndESIndexTest { @Test // @Ignore - public void uploadADSIsAboutTest() throws Exception { + public void uploadDIMEDigitalImageTest() throws Exception { boolean isRecord = true; - String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DCDF395A-2CC8-3FC2-B9A5-5A924090DF10"; - String datasource = "ads"; - String collectionId = "3"; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/DIME/F9398BD5-DECB-3FF5-8077-D96893D0D0C6"; + String datasource = "aarhusdime"; + String collectionId = "test"; readAndIndexTest(isRecord, recordId, datasource, collectionId); } From 21c6eb06fd597eeeb5f35286d1be11903b016a39 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Fri, 16 Jul 2021 18:58:48 +0200 Subject: [PATCH 13/13] fix polygon retrieving for dans and snd --- .../ariadneplus/sparql/read_collection_data_template.sparql | 2 -- .../dnetlib/ariadneplus/sparql/read_record_data_template.sparql | 2 -- 2 files changed, 4 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql index 8953445..44fc7c7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -250,8 +250,6 @@ where { %record aocat:has_spatial_coverage ?spatialRegionWKTPolygon . ?spatialRegionWKTPolygon rdf:type aocat:AO_Spatial_Region_Polygon . ?spatialRegionWKTPolygon aocat:has_polygonal_representation ?wkt . - filter (datatype(?wkt) = rdf:XMLLiteral) - filter (!isIRI(?wkt)) } } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql index 58704dc..09e6d56 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -249,8 +249,6 @@ where { %record aocat:has_spatial_coverage ?spatialRegionWKTPolygon . ?spatialRegionWKTPolygon rdf:type aocat:AO_Spatial_Region_Polygon . ?spatialRegionWKTPolygon aocat:has_polygonal_representation ?wkt . - filter (datatype(?wkt) = rdf:XMLLiteral) - filter (!isIRI(?wkt)) } } }