From 9737184b1ea29d3bd5ad970e4b65d576dd87915f Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 7 Jul 2020 13:39:22 +0200 Subject: [PATCH] start aligning es json fields to es mapping --- .../ariadneplus/elasticsearch/BulkUpload.java | 24 ++++++++++- .../elasticsearch/model/AgentInfo.java | 2 +- .../model/AriadneCatalogEntry.java | 24 ++++++++--- .../elasticsearch/model/AriadneGeoPoint.java | 40 +++++++++---------- .../elasticsearch/model/Spatial.java | 12 ++++-- .../reader/RunSPARQLQueryService.java | 7 ++-- .../src/main/resources/application.properties | 2 +- .../GraphDbReaderAndESIndexTest.java | 4 +- 8 files changed, 77 insertions(+), 38 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 294e29c..cf4b20f 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -2,6 +2,7 @@ package eu.dnetlib.ariadneplus.elasticsearch; import eu.dnetlib.ariadneplus.elasticsearch.model.AgentInfo; import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry; +import eu.dnetlib.ariadneplus.elasticsearch.model.Spatial; import eu.dnetlib.ariadneplus.reader.ResourceManager; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -19,6 +20,7 @@ import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import java.io.IOException; import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; @Service public class BulkUpload { @@ -49,17 +51,37 @@ public class BulkUpload { } - public void index(ResourceManager manager) { + public void index(ResourceManager manager, boolean isCollection) { BulkRequest request = new BulkRequest(); while (manager.hasNext()){ try { Object next = manager.next(); AriadneCatalogEntry ace = ((AriadneCatalogEntry) next); + if (isCollection) { + ace.setResourceType("collection"); + } + else { + ace.setResourceType("dataset"); + } + Spatial esSpatial = new Spatial(); + ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).forEach(s -> { + esSpatial.setPlaceName(s.getPlaceName()); + }); + ace.getSpatial().stream().filter(s -> s.getLocation()!=null).forEach(s -> { + esSpatial.setLocation(s.getLocation()); + }); + ace.getSpatial().clear(); + ace.setSpatial(Arrays.asList(esSpatial)); + String uniqueIsPartOf = ace.getUniqueIsPartOf(); + if (uniqueIsPartOf!=null) { + ace.setIsPartOf(Arrays.asList(uniqueIsPartOf)); + } AgentInfo testPublisher = new AgentInfo(); testPublisher.setName("TEST"); ace.getPublisher().add(testPublisher); String[] splits = ace.getIdentifier().split("/"); + log.debug("indexing: "+ace.toJson()); request.add(new IndexRequest(elasticSearchIndexName).id(splits[splits.length-1]) .source(ace.toJson(),XContentType.JSON)); log.debug("Indexing to ES: "+ace.toJson()); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java index b05a8bb..08e9e17 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java @@ -9,7 +9,7 @@ public class AgentInfo { private String email; private String name; private String phone; - private String type; + private String type = new String(""); public AgentInfo() { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java index 5467e33..805d9d7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java @@ -22,7 +22,8 @@ public class AriadneCatalogEntry { private List hasItemMetadataStructure; private List hasMetadataRecord; private String identifier; - private String isPartOf; + private List isPartOf; + private transient String uniqueIsPartOf; private String issued; private List keyword; private String landingPage; @@ -32,10 +33,10 @@ public class AriadneCatalogEntry { private List nativeSubject; private String originalId; private List owner; - private String packageId; + private String packageId = new String("");; private String placeName; private String postcode; - private String providerId; + private String providerId = new String("");; private String rdfAbout; private String resourceType; private String rights; @@ -176,11 +177,11 @@ public class AriadneCatalogEntry { this.identifier = identifier; } - public String getIsPartOf() { + public List getIsPartOf() { return isPartOf; } - public void setIsPartOf(String isPartOf) { + public void setIsPartOf(List isPartOf) { this.isPartOf = isPartOf; } @@ -213,7 +214,12 @@ public class AriadneCatalogEntry { } public void setLanguage(String language) { - this.language = language; + if (language!=null && language.equals("eng")) { + this.language = "en"; + } + else { + this.language = language; + } } public List getLegalResponsible() { @@ -349,7 +355,13 @@ public class AriadneCatalogEntry { this.temporal = temporal; } + public String getUniqueIsPartOf() { + return uniqueIsPartOf; + } + public void setUniqueIsPartOf(String uniqueIsPartOf) { + this.uniqueIsPartOf = uniqueIsPartOf; + } public static AriadneCatalogEntry fromJson(String json){ return new Gson().fromJson(json, AriadneCatalogEntry.class); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java index c8ae99a..b37cf0c 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java @@ -6,22 +6,22 @@ import com.google.gson.JsonElement; import java.util.Map; public class AriadneGeoPoint { - private String lat; - private String lon; + private float lat; + private float lon; - public String getLat() { + public float getLat() { return lat; } - public void setLat(String lat) { + public void setLat(float lat) { this.lat = lat; } - public String getLon() { + public float getLon() { return lon; } - public void setLon(String lon) { + public void setLon(float lon) { this.lon = lon; } @@ -32,18 +32,18 @@ public class AriadneGeoPoint { return new Gson().fromJson(json, AriadneGeoPoint.class); } - public static AriadneGeoPoint fromRDFJson(JsonElement json){ - AriadneGeoPoint agp = new AriadneGeoPoint(); - for (Map.Entry stringJsonElementEntry : json.getAsJsonObject().entrySet()) { - switch (stringJsonElementEntry.getKey()){ - case "https://www.ariadne-infrastructure.eu/property/lat": - agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "https://www.ariadne-infrastructure.eu/property/lon": - agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - } - } - return agp; - } +// public static AriadneGeoPoint fromRDFJson(JsonElement json){ +// AriadneGeoPoint agp = new AriadneGeoPoint(); +// for (Map.Entry stringJsonElementEntry : json.getAsJsonObject().entrySet()) { +// switch (stringJsonElementEntry.getKey()){ +// case "https://www.ariadne-infrastructure.eu/property/lat": +// agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "https://www.ariadne-infrastructure.eu/property/lon": +// agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// } +// } +// return agp; +// } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java index d1d02b7..530606b 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java @@ -12,8 +12,8 @@ public class Spatial { private String boundingBoxMaxLon; private String boundingBoxMinLat; private String boundingBoxMinLon; - private String coordinateSystem; - private String country; + private String coordinateSystem = new String("");; + private String country = new String("");; private AriadneGeoPoint location; private String placeName; private String lat; @@ -95,14 +95,18 @@ public class Spatial { if (this.getLocation()==null) { this.setLocation(new AriadneGeoPoint()); } - this.getLocation().setLat(lat); + if (lat!=null) { + this.getLocation().setLat(Float.parseFloat(lat)); + } } public void setLon(String lon) { if (this.getLocation()==null) { this.setLocation(new AriadneGeoPoint()); } - this.getLocation().setLon(lon); + if (lon!=null) { + this.getLocation().setLon(Float.parseFloat(lon)); + } } public Spatial() { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index fd79685..6e20098 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -71,8 +71,9 @@ public class RunSPARQLQueryService { private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){ log.debug("Retrieving "+recordId+" - isCollection:"+isCollection ); - String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">"); - openConnection(); + String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">"); + log.debug(query); + openConnection(); StringWriter recordWriter = null; Model resultsModel = null; String jsonRecord = null; @@ -95,7 +96,7 @@ public class RunSPARQLQueryService { } parser.parse(recordWriter.toString()); resourceManager.manage(parser); - bulkUpload.index(resourceManager); + bulkUpload.index(resourceManager, isCollection); } catch(Exception e){ e.printStackTrace(); } finally{ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index 09399de..93518af 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -310,7 +310,7 @@ class.map.specifications={\ "element_type": "java.lang.String"\ },\ "https://www.ariadne-infrastructure.eu/property/partOf": {\ -"class_field": "IsPartOf",\ +"class_field": "UniqueIsPartOf",\ "substring": "no",\ "element_type": "java.lang.String"\ }\ diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 82b6154..f7d5d2a 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -52,13 +52,13 @@ public class GraphDbReaderAndESIndexTest { BulkUpload bulkUpload = new BulkUpload(); bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname")); runSPQRLQuery.setBulkUpload(bulkUpload); - String recordId = ""; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0320AA03-BED9-3F2B-AF5D-E31EE50CAE97"; String datasource = "ads"; String collectionId = "271"; List recordIds = Arrays.asList(recordId); final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); - runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId); + runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false); }