diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index 2d5c6d9..294e29c 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -1,10 +1,10 @@ package eu.dnetlib.ariadneplus.elasticsearch; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; - -import javax.annotation.PostConstruct; - +import eu.dnetlib.ariadneplus.elasticsearch.model.AgentInfo; +import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry; +import eu.dnetlib.ariadneplus.reader.ResourceManager; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.http.HttpHost; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -16,34 +16,59 @@ import org.elasticsearch.common.xcontent.XContentType; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry; -import eu.dnetlib.ariadneplus.reader.ResourceManager; +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; @Service public class BulkUpload { - @Value("${elasticsearch.url:localhost:9200}") - private String elasticsearchUrl; + private static final Log log = LogFactory.getLog(BulkUpload.class); + + @Value("${elasticsearch.hostname}") + private String elasticSearchHostName; + + @Value("${elasticsearch.indexname}") + private String elasticSearchIndexName; private RestHighLevelClient client; - @PostConstruct - private void init() throws IOException { + public void init(String elasticSearchHostName, String elasticSearchIndexName) throws IOException { + this.elasticSearchIndexName = elasticSearchIndexName; client = new RestHighLevelClient( RestClient.builder( - new HttpHost("localhost",9200,"http"))); + new HttpHost(elasticSearchHostName,9200,"http"))); + + } + + @PostConstruct + public void init() throws IOException { + client = new RestHighLevelClient( + RestClient.builder( + new HttpHost(elasticSearchHostName,9200,"http"))); } public void index(ResourceManager manager) { BulkRequest request = new BulkRequest(); while (manager.hasNext()){ + try { - AriadneCatalogEntry ace = ((AriadneCatalogEntry) manager.next()); - request.add(new IndexRequest("prova_via_code").id(ace.getOriginalId()) + Object next = manager.next(); + AriadneCatalogEntry ace = ((AriadneCatalogEntry) next); + AgentInfo testPublisher = new AgentInfo(); + testPublisher.setName("TEST"); + ace.getPublisher().add(testPublisher); + String[] splits = ace.getIdentifier().split("/"); + request.add(new IndexRequest(elasticSearchIndexName).id(splits[splits.length-1]) .source(ace.toJson(),XContentType.JSON)); - System.out.println("indexing to ES record "+ace.getOriginalId()); + log.debug("Indexing to ES: "+ace.toJson()); BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + log.info("Indexing to ES completed with status: "+bulkResponse.status()); + if (bulkResponse.hasFailures()) { + log.error("FailureMessage: "+bulkResponse.buildFailureMessage()); + } + } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (NoSuchMethodException e) { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java index 9a885a1..b05a8bb 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java @@ -55,16 +55,16 @@ public class AgentInfo { AgentInfo pi = new AgentInfo(); for (Map.Entry entry : json.getAsJsonObject().entrySet()){ switch (entry.getKey()){ - case "http://www.myprefix/name" : + case "https://www.ariadne-infrastructure.eu/property/name" : pi.setName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/type": + case "https://www.ariadne-infrastructure.eu/property/type": pi.setType(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/email": + case "https://www.ariadne-infrastructure.eu/property/email": pi.setEmail(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/phone": + case "https://www.ariadne-infrastructure.eu/property/phone": pi.setPhone(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java new file mode 100644 index 0000000..ae7cce6 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArchaeologicalResourceType.java @@ -0,0 +1,52 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + +public class ArchaeologicalResourceType { + private long id = 100000; + private String name; + + public ArchaeologicalResourceType() { + } + +// public static ArchaeologicalResourceType fromRDFJson(JsonElement json) { +// ArchaeologicalResourceType art = new ArchaeologicalResourceType(); +// for (Map.Entry entry : json.getAsJsonObject().entrySet()){ +// switch (entry.getKey()){ +// case "https://www.ariadne-infrastructure.eu/property/id" : +// art.setId(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "https://www.ariadne-infrastructure.eu/property/name": +// String tmp = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); +// art.setName(tmp); +// break; +// +// } +// } +// +// return art; +// } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public static ArchaeologicalResourceType fromJson(String json){ + return new Gson().fromJson(json, ArchaeologicalResourceType.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java deleted file mode 100644 index 80ea18c..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java +++ /dev/null @@ -1,52 +0,0 @@ -package eu.dnetlib.ariadneplus.elasticsearch.model; - -import com.google.gson.Gson; -import com.google.gson.JsonElement; - -import java.util.Map; - -public class ArcheologicalResourceType { - private String id; - private String name; - - public ArcheologicalResourceType() { - } - - public static ArcheologicalResourceType fromRDFJson(JsonElement json) { - ArcheologicalResourceType art = new ArcheologicalResourceType(); - for (Map.Entry entry : json.getAsJsonObject().entrySet()){ - switch (entry.getKey()){ - case "http://www.myprefix/id" : - art.setId(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/name": - String tmp = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); - art.setName(tmp); - break; - - } - } - - return art; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public static ArcheologicalResourceType fromJson(String json){ - return new Gson().fromJson(json, ArcheologicalResourceType.class); - } -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java index 938868f..5467e33 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java @@ -1,14 +1,15 @@ package eu.dnetlib.ariadneplus.elasticsearch.model; import com.google.gson.Gson; -import java.util.List; +import eu.dnetlib.ariadneplus.reader.utils.ESUtils; +import java.util.List; public class AriadneCatalogEntry { private List aatSubjects; private String accessPolicy; private String accessRights; - private ArcheologicalResourceType archeologicalResourceType; + private ArchaeologicalResourceType archaeologicalResourceType; private String contactPoint; private List contributor; private List creator; @@ -21,7 +22,7 @@ public class AriadneCatalogEntry { private List hasItemMetadataStructure; private List hasMetadataRecord; private String identifier; - private String partOf; + private String isPartOf; private String issued; private List keyword; private String landingPage; @@ -40,6 +41,8 @@ public class AriadneCatalogEntry { private String rights; private List scientificResponsible; private List spatial; +// private List spatialRegion; +// private List spatialRegionPoint; private List technicalResponsible; private List temporal; @@ -77,12 +80,12 @@ public class AriadneCatalogEntry { this.accessRights = accessRights; } - public ArcheologicalResourceType getArcheologicalResourceType() { - return archeologicalResourceType; + public ArchaeologicalResourceType getArchaeologicalResourceType() { + return archaeologicalResourceType; } - public void setArcheologicalResourceType(ArcheologicalResourceType archeologicalResourceType) { - this.archeologicalResourceType = archeologicalResourceType; + public void setArchaeologicalResourceType(ArchaeologicalResourceType archaeologicalResourceType) { + this.archaeologicalResourceType = archaeologicalResourceType; } public String getContactPoint() { @@ -173,12 +176,12 @@ public class AriadneCatalogEntry { this.identifier = identifier; } - public String getPartOf() { - return partOf; + public String getIsPartOf() { + return isPartOf; } - public void setPartOf(String partOf) { - this.partOf = partOf; + public void setIsPartOf(String isPartOf) { + this.isPartOf = isPartOf; } public String getIssued() { @@ -186,7 +189,7 @@ public class AriadneCatalogEntry { } public void setIssued(String issued) { - this.issued = issued; + this.issued = ESUtils.getESFormatDate(issued); } public List getKeyword() { @@ -226,7 +229,7 @@ public class AriadneCatalogEntry { } public void setModified(String modified) { - this.modified = modified; + this.modified = ESUtils.getESFormatDate(modified); } public List getNativeSubject() { @@ -322,7 +325,12 @@ public class AriadneCatalogEntry { } public void setSpatial(List spatial) { - this.spatial = spatial; + if (this.spatial==null) { + this.spatial = spatial; + } + else { + this.spatial.addAll(spatial); + } } public List getTechnicalResponsible() { @@ -350,104 +358,4 @@ public class AriadneCatalogEntry { public String toJson(){ return new Gson().toJson(this); } - -// public static AriadneCatalogEntry fromRDFJson(JsonElement json, String identifier, Map map){ -// AriadneCatalogEntry acim = new AriadneCatalogEntry(); -// acim.setIdentifier(identifier.substring(identifier.lastIndexOf("/") + 1)); -// JsonObject content = json.getAsJsonObject(); -// for (Map.Entry stringJsonElementEntry : content.entrySet()) { -// switch (stringJsonElementEntry.getKey()){ -// case "http://www.myprefix/accessPolicy": -// acim.setAccessPolicy(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/accessRights": -// acim.setAccessRights(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/contributor": -// JsonArray contributor_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List contributor_list = new ArrayList(); -// for (int i = 0; i < contributor_array.size() ; i++ ){ -// String map_key = contributor_array.get(i).getAsJsonObject().get("value").getAsString(); -// contributor_list.add(AgentInfo.fromRDFJson(map.get(map_key))); -// -// } -// acim.setContributor(contributor_list); -// break; -// case "http://www.myprefix/description": -// acim.setDescription(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/isPartOf": -// acim.setPartOf(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/issued": -// acim.setIssued(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/landingPage": -// acim.setLandingPage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/language": -// acim.setLanguage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/modified": -// acim.setModified(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/nativeSubject": -// JsonArray nativeSubject_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List nativeSubject_list = new ArrayList(); -// for (int i = 0; i < nativeSubject_array.size() ; i++ ){ -// String map_key = nativeSubject_array.get(i).getAsJsonObject().get("value").getAsString(); -// nativeSubject_list.add(NativeSubject.fromRDFJson(map.get(map_key))); -// -// } -// acim.setNativeSubject(nativeSubject_list); -// break; -// case "http://www.myprefix/originalId": -// acim.setOriginalId(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/resourceType": -// acim.setResourceType(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/spatial": -// JsonArray spatial_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List spatial_list = new ArrayList(); -// for (int i = 0; i < spatial_array.size() ; i++ ){ -// String map_key = spatial_array.get(i).getAsJsonObject().get("value").getAsString(); -// spatial_list.add(Spatial.fromRDFJson(map.get(map_key), map)); -// -// } -// acim.setSpatial(spatial_list); -// break; -// case "http://www.myprefix/temporal": -// JsonArray temporal_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List temporal_list = new ArrayList<>(); -// for(int i=0; i < temporal_array.size(); i++){ -// String map_key = temporal_array.get(i).getAsJsonObject().get("value").getAsString(); -// temporal_list.add(AriadneTemporal.fromRDFJson(map.get(map_key))); -// } -// acim.setTemporal(temporal_list); -// break; -// case "http://www.myprefix/title": -// acim.setTitle(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); -// break; -// case "http://www.myprefix/publisher": -// JsonArray publisher_array = stringJsonElementEntry.getValue().getAsJsonArray(); -// List publisher_list = new ArrayList(); -// for (int i = 0; i < publisher_array.size() ; i++ ){ -// String map_key = publisher_array.get(i).getAsJsonObject().get("value").getAsString(); -// publisher_list.add(AgentInfo.fromRDFJson(map.get(map_key))); -// -// } -// acim.setPublisher(publisher_list); -// break; -// case "http://www.myprefix/archeologicalResourceType": -// acim.setArcheologicalResourceType(ArcheologicalResourceType.fromRDFJson(map.get(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()))); -// } -// -// -// } -// -// System.out.println(acim.toJson()); -// return acim; -// } -} -//https://ariadne-infrastructure.eu/aocat \ No newline at end of file +} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java index 2b6b295..c8ae99a 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java @@ -36,10 +36,10 @@ public class AriadneGeoPoint { AriadneGeoPoint agp = new AriadneGeoPoint(); for (Map.Entry stringJsonElementEntry : json.getAsJsonObject().entrySet()) { switch (stringJsonElementEntry.getKey()){ - case "http://www.myprefix/lat": + case "https://www.ariadne-infrastructure.eu/property/lat": agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/lon": + case "https://www.ariadne-infrastructure.eu/property/lon": agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java index 115b4fa..7dd30cc 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java @@ -15,16 +15,16 @@ public class AriadneTemporal { AriadneTemporal at = new AriadneTemporal(); for (Map.Entry entry : json.getAsJsonObject().entrySet()){ switch (entry.getKey()){ - case "http://www.myprefix/from" : + case "https://www.ariadne-infrastructure.eu/property/from" : at.setFrom(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/periodName": + case "https://www.ariadne-infrastructure.eu/property/periodName": at.setPeriodName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/until": + case "https://www.ariadne-infrastructure.eu/property/until": at.setUntil(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/uri": + case "https://www.ariadne-infrastructure.eu/property/uri": at.setUri(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java index 324eca8..a211d37 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java @@ -38,10 +38,10 @@ public class NativeSubject { NativeSubject pi = new NativeSubject(); for (Map.Entry entry : json.getAsJsonObject().entrySet()){ switch (entry.getKey()){ - case "http://www.myprefix/prefLabel" : + case "https://www.ariadne-infrastructure.eu/property/prefLabel" : pi.setPrefLabel(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; - case "http://www.myprefix/rdfAbout": + case "https://www.ariadne-infrastructure.eu/property/rdfAbout": pi.setRdfAbout(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); break; diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java index 0587975..d1d02b7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java @@ -16,44 +16,8 @@ public class Spatial { private String country; private AriadneGeoPoint location; private String placeName; - - public static Spatial fromRDFJson(JsonElement json, Map map) { - Spatial pi = new Spatial(); - for (Map.Entry entry : json.getAsJsonObject().entrySet()){ - switch (entry.getKey()){ - case "http://www.myprefix/address" : - pi.setAddress(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/boundingBoxMaxLat": - pi.setBoundingBoxMaxLat(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/boundingBoxMaxLon": - pi.setBoundingBoxMaxLon(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/boundingBoxMinLat": - pi.setBoundingBoxMinLat(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/boundingBoxMinLon": - pi.setBoundingBoxMinLon(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/country": - pi.setCountry(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/coordinateSystem": - pi.setCoordinateSystem(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - case "http://www.myprefix/location": - String map_key = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); - pi.setLocation(AriadneGeoPoint.fromRDFJson(map.get(map_key))); - break; - case "http://www.myprefix/placeName": - pi.setPlaceName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); - break; - } - } - - return pi; - } + private String lat; + private String lon; public String getAddress() { return address; @@ -127,6 +91,20 @@ public class Spatial { this.placeName = placeName; } + public void setLat(String lat) { + if (this.getLocation()==null) { + this.setLocation(new AriadneGeoPoint()); + } + this.getLocation().setLat(lat); + } + + public void setLon(String lon) { + if (this.getLocation()==null) { + this.setLocation(new AriadneGeoPoint()); + } + this.getLocation().setLon(lon); + } + public Spatial() { } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java index f2d67af..d685986 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java @@ -2,8 +2,16 @@ package eu.dnetlib.ariadneplus.graphdb; import java.io.File; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.ResourceManager; +import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -23,19 +31,32 @@ import org.eclipse.rdf4j.rio.RDFFormat; import eu.dnetlib.ariadneplus.publisher.AriadnePlusPublisherException; import eu.dnetlib.ariadneplus.rdf.RecordParserHelper; import net.sf.saxon.s9api.SaxonApiException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Component; /** * @author enrico.ottonello * */ +@Component public class GraphDBClient { private static final Log log = LogFactory.getLog(GraphDBClient.class); public static final String PROVENANCE_NS = "http://www.d-net.research-infrastructures.eu/provenance/"; - public static final int NUM_RECORDS_THRESHOLD = 10; - + public static final int NUM_RECORDS_THRESHOLD = 10; + + @Autowired + private RunSPARQLQueryService runSPQRLQuery; + @Autowired + private ParseRDFJSON parseRDFJSON; + @Autowired + private ResourceManager resourceManager; + @Autowired + private BulkUpload bulkUpload; + private RecordParserHelper recordParserHelper; private String graphDBServerUrl; private String graphDBBaseURI; @@ -43,7 +64,7 @@ public class GraphDBClient { private String writerPwd; private String repository; - protected GraphDBClient(final RecordParserHelper recordParserHelper, + protected void setup(final RecordParserHelper recordParserHelper, final String graphDBServerUrl, final String graphDBBaseURI, final String writerUser, final String writerPwd, final String repository) { this.recordParserHelper = recordParserHelper; this.graphDBServerUrl = graphDBServerUrl; @@ -318,4 +339,37 @@ public class GraphDBClient { throw new AriadnePlusPublisherException(e); } } + + public RunSPARQLQueryService getRunSPQRLQuery() { + return runSPQRLQuery; + } + + public void setRunSPQRLQuery(RunSPARQLQueryService runSPQRLQuery) { + this.runSPQRLQuery = runSPQRLQuery; + } + + public String indexOnES(String datasource, String collectionId) throws AriadnePlusPublisherException { + try { + runSPQRLQuery.setupConnection( getWriterUser(), getWriterPwd(), this.graphDBServerUrl, getRepository()); + runSPQRLQuery.setParser(parseRDFJSON); + runSPQRLQuery.setResourceManager(resourceManager); + runSPQRLQuery.setBulkUpload(bulkUpload); +// String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/02E4F4B5-24B7-3AD7-B460-CFA8B1F0BD1F"; +// List recordIds = Arrays.asList(recordId); + List recordIds = runSPQRLQuery.selectRecordIds(datasource, collectionId); + final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); + String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); + boolean isCollection = false; + runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, isCollection); + List collectionResourceId = runSPQRLQuery.selectCollectionId(datasource, collectionId); + final ClassPathResource selectCollectionTemplateRes = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql"); + String selectCollectionTemplate = IOUtils.toString(selectCollectionTemplateRes.getInputStream(), StandardCharsets.UTF_8.name()); + isCollection = true; + runSPQRLQuery.executeMultipleQueryGraph(selectCollectionTemplate, collectionResourceId, datasource, collectionId, isCollection); + }catch(Throwable e){ + log.error(e); + throw new AriadnePlusPublisherException(e); + } + return "ok"; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClientFactory.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClientFactory.java index 701ec99..78805b6 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClientFactory.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClientFactory.java @@ -31,9 +31,12 @@ public class GraphDBClientFactory { @Autowired private RecordParserHelper recordParserHelper; + @Autowired + private GraphDBClient graphDBClient; + public GraphDBClient getGraphDBClient() { - log.debug("Creating GraphDBClient for "+graphDBServerUrl); - return new GraphDBClient(recordParserHelper, graphDBServerUrl, graphDBBaseURI, writerUser, writerPwd, repository); + graphDBClient.setup(recordParserHelper, graphDBServerUrl, graphDBBaseURI, writerUser, writerPwd, repository); + return graphDBClient; } public RecordParserHelper getRecordParserHelper() { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java index 91c6454..8d696da 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java @@ -69,4 +69,8 @@ public class AriadnePlusPublisherController { return getAriadnePlusPublisherHelper().feedFromURL(dataUrl, context, getTarget(DEFAULT_TARGET_ENDPOINT)); } + @RequestMapping(value = "/indexOnES", method = RequestMethod.POST) + public String indexOnES(@RequestParam final String datasource, @RequestParam final String collectionId) throws AriadnePlusPublisherException { + return getAriadnePlusPublisherHelper().indexOnES(datasource, collectionId, getTarget(DEFAULT_TARGET_ENDPOINT)); + } } \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java index 4c5fcf4..eed6ac0 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java @@ -88,6 +88,17 @@ public class AriadnePlusPublisherHelper { return res; } + public String indexOnES(final String datasource, final String collectionId, final AriadnePlusTargets target) throws AriadnePlusPublisherException { + String res; + switch(target){ + case GRAPHDB: + res = indexOnES(datasource, collectionId); + break; + default: throw new AriadnePlusPublisherException("Target "+target+" not supported yet"); + } + return res; + } + private void publishGraphDB(final String record) throws AriadnePlusPublisherException { log.debug("Publishing on graphdb"); GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); @@ -124,4 +135,10 @@ public class AriadnePlusPublisherHelper { GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); return graphDBClient.feedFromURL(dataUrl, context); } + + private String indexOnES(final String datasource, final String collectionId) throws AriadnePlusPublisherException { + log.info("indexOnES "+datasource + " " + collectionId); + GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); + return graphDBClient.indexOnES(datasource, collectionId); + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java index 732fe78..73b2196 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java @@ -10,37 +10,55 @@ import java.util.Set; import javax.annotation.PostConstruct; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import eu.dnetlib.ariadneplus.reader.utils.ClassSpec; +import eu.dnetlib.ariadneplus.reader.utils.Mappings; +import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; -import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; -import eu.dnetlib.ariadneplus.reader.utils.ClassSpec; -import eu.dnetlib.ariadneplus.reader.utils.Mappings; -import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap; import net.minidev.json.JSONArray; import net.minidev.json.JSONObject; @Service public class ResourceManager { + private static final Log log = LogFactory.getLog(ResourceManager.class); + @Value("${type.path:undefined}") private String type_path; @Value("${general.classpath:undefined}") private String general_classpath; @Value("${exclude.predicates:[]}") private String exclude_predicates; + @Value("${class.map.specifications:undefined}") + private String spec; private List not_parsable; private ParseRDFJSON parser; + private PropertiesMap propertiesMap; + + public void setup(String type_path, String general_classpath, String exclude_predicates, String spec) { + this.type_path = type_path; + this.general_classpath = general_classpath; + this.exclude_predicates = exclude_predicates; + this.spec = spec; + init(); + } + @PostConstruct public void init(){ Type listType = new TypeToken>(){}.getType(); not_parsable = new Gson().fromJson(exclude_predicates, listType); + propertiesMap = new PropertiesMap(); + propertiesMap.fill(spec); } @@ -69,11 +87,17 @@ public class ResourceManager { if(entry instanceof LinkedHashMap){ LinkedHashMap tmp = (LinkedHashMap)((JSONArray)((LinkedHashMap)entry).get(type_path)).get(0); class_name = (String)tmp.get("value"); + if (class_name.equals("provided record") || class_name.equals("Dataset Collection")) { + class_name = "AriadneCatalogEntry"; + } } } + if (entry == null) { + return null; + } Class c = Class.forName(general_classpath + class_name); Object class_instance = c.newInstance(); - ClassSpec class_spec = PropertiesMap.get(class_name); + ClassSpec class_spec = propertiesMap.get(class_name); Set keySet; if(entry instanceof LinkedHashMap) @@ -101,7 +125,7 @@ public class ResourceManager { setField.invoke(class_instance, getFieldValue(values.get(0))); } else{ - if(PropertiesMap.get(map.getExternal_reference()).getClass_type().equals("prototype")){ + if(propertiesMap.get(map.getExternal_reference()).getClass_type().equals("prototype")){ List value_list = new ArrayList<>(); for(Object value: values){ value_list.add(manage(ParseRDFJSON.get(getFieldValue(value)), map.getExternal_reference())); @@ -122,6 +146,35 @@ public class ResourceManager { return class_instance; } + public String getType_path() { + return type_path; + } + public void setType_path(String type_path) { + this.type_path = type_path; + } + public String getGeneral_classpath() { + return general_classpath; + } + + public void setGeneral_classpath(String general_classpath) { + this.general_classpath = general_classpath; + } + + public String getExclude_predicates() { + return exclude_predicates; + } + + public void setExclude_predicates(String exclude_predicates) { + this.exclude_predicates = exclude_predicates; + } + + public String getSpec() { + return spec; + } + + public void setSpec(String spec) { + this.spec = spec; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index 749ec67..fd79685 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -1,37 +1,28 @@ package eu.dnetlib.ariadneplus.reader; -import java.io.StringWriter; - -import org.eclipse.rdf4j.model.IRI; -import org.eclipse.rdf4j.model.Literal; +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.Resource; -import org.eclipse.rdf4j.model.ValueFactory; -import org.eclipse.rdf4j.model.impl.SimpleValueFactory; -import org.eclipse.rdf4j.query.GraphQuery; -import org.eclipse.rdf4j.query.GraphQueryResult; -import org.eclipse.rdf4j.query.QueryLanguage; -import org.eclipse.rdf4j.query.QueryResults; +import org.eclipse.rdf4j.query.*; import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryConnection; import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; import org.eclipse.rdf4j.rio.RDFFormat; import org.eclipse.rdf4j.rio.RDFWriter; import org.eclipse.rdf4j.rio.Rio; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; -import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; @Service public class RunSPARQLQueryService { - @Value("${sparql.query3:undefined}") - private String query3; - - @Value("${repository.url:undefined}") - private String repository_url; + private static final Log log = LogFactory.getLog(RunSPARQLQueryService.class); private RepositoryConnection connection; private RemoteRepositoryManager manager; @@ -41,12 +32,23 @@ public class RunSPARQLQueryService { private ResourceManager resourceManager; private BulkUpload bulkUpload; - private void openConnection(){ - manager = new RemoteRepositoryManager(repository_url); + private static String username = null; + private static String pwd = null; + private static String graphDBUrl = null; + private static String graphDBRepository = null; + + public void setupConnection(String username, String pwd, String graphDbUrl, String graphDbRepository) { + setUsername(username); + setPwd(pwd); + setGraphDBUrl(graphDbUrl); + setGraphDBRepository(graphDbRepository); + } + + private void openConnection(){ + manager = new RemoteRepositoryManager(getGraphDBUrl()); manager.init(); - manager.setUsernameAndPassword("writer", "Writer01"); -// repository = manager.getRepository("test01"); - repository = manager.getRepository("ariadneplus-ts01"); + manager.setUsernameAndPassword(getUsername(), getPwd()); + repository = manager.getRepository(getGraphDBRepository()); connection = repository.getConnection(); } @@ -57,86 +59,156 @@ public class RunSPARQLQueryService { manager.shutDown(); } - public String executeQueryGraph(){ - if (query3.equals("undefined")) - return null; - openConnection(); + public String executeMultipleQueryGraph(String queryTemplate, List recordIds, String datasource, String collectionId, boolean isCollection){ + if (queryTemplate==null) + return null; + final String selectQueryTemplate = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); + recordIds.forEach(recordId -> { + executeQueryGraph(selectQueryTemplate, recordId, isCollection); + }); + return "ok"; + } + + private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){ + log.debug("Retrieving "+recordId+" - isCollection:"+isCollection ); + String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">"); + openConnection(); StringWriter recordWriter = null; - Model model = null; + Model resultsModel = null; String jsonRecord = null; try { - GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query3); + log.debug("Started at: "+Calendar.getInstance().getTime().toString()); + GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query); GraphQueryResult graphQueryResult = graphQuery.evaluate(); - System.out.println("as model ..."); - model = QueryResults.asModel(graphQueryResult); + resultsModel = QueryResults.asModel(graphQueryResult); graphQueryResult.close(); - int resourceCount = 0; - System.out.println("counting resources ... " ); - ValueFactory factory = SimpleValueFactory.getInstance(); - IRI iri = factory.createIRI("http://www.myprefix/resourcetype"); - Literal value = factory.createLiteral("AriadneCatalogEntry"); - for (Resource record: model.filter(null, iri, value).subjects()) { - System.out.println(); - Model recordModel = null; -// RDFWriter rdfRecordWriter = null; - resourceCount+=1; -// if (resourceCount==12) { -// break; -// } - recordModel = model.filter(record, null, null); - if (recordModel!=null && !recordModel.isEmpty()) { - recordWriter = new StringWriter(); - RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); - Rio.write(recordModel, rdfRecordWriter); -// System.out.println("record json: "+ recordWriter.toString()); - parser.parse(recordWriter.toString()); - resourceManager.manage(parser); - bulkUpload.index(resourceManager); - } - } - System.out.println(); - System.out.println("count resources: "+ resourceCount); - System.out.println("count statements: " + model.size()); - System.out.println("index to Elastic Search completed"); - }catch(Exception e){ + log.debug("Finished at: "+Calendar.getInstance().getTime().toString()); + log.debug("Statements retrieved: " + resultsModel.size()); + if (resultsModel.size()==0) { + return "noresult"; + } + recordWriter = new StringWriter(); + RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); + Rio.write(resultsModel, rdfRecordWriter); + if (isCollection) { + parser.setCollection(true); + } + parser.parse(recordWriter.toString()); + resourceManager.manage(parser); + bulkUpload.index(resourceManager); + } catch(Exception e){ e.printStackTrace(); - }finally{ + } finally{ closeConnection(); - if (model!=null) { - model.clear(); + if (resultsModel!=null) { + resultsModel.clear(); } } return jsonRecord; } - public ParseRDFJSON getParser() { return parser; } - public void setParser(ParseRDFJSON parser) { this.parser = parser; } - public ResourceManager getResourceManager() { return resourceManager; } - public void setResourceManager(ResourceManager resourceManager) { this.resourceManager = resourceManager; } - public BulkUpload getBulkUpload() { return bulkUpload; } - public void setBulkUpload(BulkUpload bulkUpload) { this.bulkUpload = bulkUpload; } + public static String getUsername() { + return username; + } + + public static String getPwd() { + return pwd; + } + + public static String getGraphDBUrl() { + return graphDBUrl; + } + + public static String getGraphDBRepository() { + return graphDBRepository; + } + + public static void setUsername(String username) { + RunSPARQLQueryService.username = username; + } + + public static void setPwd(String pwd) { + RunSPARQLQueryService.pwd = pwd; + } + + public static void setGraphDBUrl(String graphDBUrl) { + RunSPARQLQueryService.graphDBUrl = graphDBUrl; + } + + public static void setGraphDBRepository(String graphDBRepository) { + RunSPARQLQueryService.graphDBRepository = graphDBRepository; + } + + public List selectRecordIds(String datasource, String collectionId){ + log.debug("Retrieving record Ids from GraphDB ..."); + String queryTemplate = "PREFIX rdf: \n" + + "select * \n" + + "from \n" + + "where { \n" + + "\t?recordId rdf:type .\n" + + "} \n"; + String query = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); + return executeSelect(query); + } + + public List selectCollectionId(String datasource, String collectionId){ + log.debug("Retrieving collection Id from GraphDB ..."); + String queryTemplate = "PREFIX rdf: \n" + + "select * \n" + + "from \n" + + "where { \n" + + "\t?recordId rdf:type .\n" + + "} \n"; + String query = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); + return executeSelect(query); + } + + private List executeSelect(String query){ + openConnection(); + String jsonRecord = null; + List results = new ArrayList<>(); + try { + log.debug("Started at: "+Calendar.getInstance().getTime().toString()); + TupleQuery selectQuery = connection.prepareTupleQuery(QueryLanguage.SPARQL, query); + TupleQueryResult selectQueryResult = selectQuery.evaluate(); + int counter = 0; + while (selectQueryResult.hasNext()) { + BindingSet recordSet = selectQueryResult.next(); + org.eclipse.rdf4j.model.Value recordIdValue = recordSet.getValue("recordId"); + results.add(recordIdValue.stringValue()); + counter++; + } + log.debug("Total records retrieved: "+counter); + log.debug("Finished at: "+Calendar.getInstance().getTime().toString()); + } catch(Exception e){ + e.printStackTrace(); + } finally{ + closeConnection(); + } + return results; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java index 89ca81a..4a2549f 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java @@ -3,6 +3,9 @@ package eu.dnetlib.ariadneplus.reader.json; import java.util.Iterator; import java.util.LinkedHashMap; +import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -18,10 +21,17 @@ import net.minidev.json.parser.ParseException; @Service public class ParseRDFJSON { + private static final Log log = LogFactory.getLog(ParseRDFJSON.class); + static JSONObject map ; - @Value("${catalog.entry.path:undefined}") - private String query; + @Value("${catalog.entry.path}") + private String catalogEntryJsonPath; + + @Value("${catalog.entry.collection.path}") + private String catalogEntryCollectionJsonPath; + + private boolean isCollection = false; private String json; @@ -45,8 +55,10 @@ public class ParseRDFJSON { setJson(json); fillMap(); DocumentContext jsonContext = JsonPath.parse(json); - JSONArray entries = jsonContext.read(query); + log.debug(getCatalogEntryJsonPath()); + JSONArray entries = jsonContext.read(getCatalogEntryJsonPath()); int size = entries.size(); + log.debug("num elements in json: "+size); it = entries.iterator(); } @@ -62,5 +74,22 @@ public class ParseRDFJSON { return (JSONObject) map.get(key); } + public String getCatalogEntryJsonPath() { + if (isCollection) { + return catalogEntryCollectionJsonPath; + } + return catalogEntryJsonPath; + } + public void setCatalogEntryJsonPath(String catalogEntryJsonPath) { + this.catalogEntryJsonPath = catalogEntryJsonPath; + } + + public boolean isCollection() { + return isCollection; + } + + public void setCollection(boolean collection) { + isCollection = collection; + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java new file mode 100644 index 0000000..fb8b0df --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java @@ -0,0 +1,14 @@ +package eu.dnetlib.ariadneplus.reader.utils; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; + +public class ESUtils { + private static DateTimeFormatter originalRecordDateFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy"); + private static DateTimeFormatter elasticSearchDateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + public static String getESFormatDate(String originalDate) { + LocalDate parsedDate = LocalDate.parse(originalDate, originalRecordDateFormatter); + return parsedDate.format(elasticSearchDateFormatter); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java index 897f3e4..a79b5f2 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java @@ -1,16 +1,14 @@ package eu.dnetlib.ariadneplus.reader.utils; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; - -import javax.annotation.PostConstruct; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - import com.google.common.reflect.TypeToken; import com.google.gson.Gson; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; @Component public class PropertiesMap { @@ -20,10 +18,16 @@ public class PropertiesMap { private static Map map; - public static ClassSpec get(String key){ + public ClassSpec get(String key){ return map.get(key); } + public void fill(String spec){ + Type mapType = new TypeToken>(){}.getType(); + map = new Gson().fromJson(spec,mapType); + + } + @PostConstruct public void fill(){ Type mapType = new TypeToken>(){}.getType(); diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index e367fe5..09399de 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -3,149 +3,40 @@ server.servlet.context-path=/ariadneplus-graphdb server.port=8281 -graphdb.serverUrl=http://localhost:7200/ +graphdb.serverUrl=http://graphdb-test.ariadne.d4science.org:7200 graphdb.writer.user=writer -graphdb.writer.pwd=writer01 -graphdb.repository=ariadneprova -graphdb.sparqlUrl = http://localhost:7200/sparql - +graphdb.writer.pwd=***** +graphdb.repository=ariadneplus-ts01 graphdb.baseURI=https://ariadne-infrastructure.eu/ -#repository.url=http://localhost:7200 -repository.url=http://graphdb-test.ariadne.d4science.org:7200 +elasticsearch.hostname=elastic-test.ariadne.d4science.org +elasticsearch.indexname=catalog_test -sparql.query2=PREFIX skos: PREFIX rdfs: PREFIX aocat: PREFIX mine: CONSTRUCT { ?archeologicalResourceType a mine:archeologicalResourceType . ?record mine:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType mine:name ?archeologicalResourceTypeName . } WHERE { GRAPH { ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . } } - -sparql.query=PREFIX aocat: \ -PREFIX rdfs: \ -PREFIX skos: \ -PREFIX mine: \ -CONSTRUCT { \ - ?record mine:resourcetype "AriadneCatalogEntry" . \ - ?record mine:identifier ?record .\ - ?record mine:publisher ?x . \ - ?x a mine:AgentInfo . \ - ?x mine:name ?publisherName . \ - ?x mine:type ?publisherType . \ - ?archeologicalResourceType a mine:archeologicalResourceType . \ -?record mine:archeologicalResourceType ?archeologicalResourceType . \ -?archeologicalResourceType mine:name ?archeologicalResourceTypeName . \ -?record mine:issued ?issued . \ -?record mine:resourceType ?resourceType . \ -?record mine:modified ?modified . \ -?record mine:nativeSubject ?ns . \ - ?ns a mine:NativeSubject . \ -?ns mine:prefLabel ?nativeSbj . \ - ?spatial a mine:Spatial . \ -?record mine:spatial ?spatial . \ -?spatial mine:placeName ?spatialPlaceName . \ -?spatial mine:location ?blocation . \ - ?blocation a mine:AriadneGeoPoint . \ -?blocation mine:lat ?spatialLocationLat . \ -?blocation mine:lon ?spatialLocationLon . \ -?record mine:accessPolicy ?accessPolicy . \ -?record mine:landingPage ?landingPage . \ -?record mine:title ?title . \ -?record mine:accessRights ?accessRights . \ -?record mine:description ?description . \ -?record mine:contributor ?contributor . \ - ?contributor a mine:AgentInfo . \ -?contributor mine:name ?contributorName . \ -?contributor mine:type ?contributorType . \ -?record mine:originalId ?originalId . \ - ?record mine:temporal ?t .\ - ?t a mine:AriadneTemporal . \ -?t mine:periodName ?temporalPeriodName . \ -?record mine:language ?language . \ -} \ -WHERE { \ -GRAPH { \ -?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . \ -?x rdfs:label ?publisherName . \ -?record aocat:has_native_subject ?ns . \ -?ns skos:prefLabel ?nativeSbj . \ -?type skos:prefLabel ?resourceType . \ -?record aocat:has_spatial_coverage ?spatial . \ -?spatial aocat:has_place_name ?spatialPlaceName . \ -?spatial aocat:has_latitude ?spatialLocationLat . \ -?spatial aocat:has_longitude ?spatialLocationLon . \ -?record aocat:has_access_policy ?accessPolicy . \ -?record aocat:has_landing_page ?z . \ -?z rdfs:label ?landingPage . \ -?record aocat:has_title ?title . \ -?record aocat:has_access_policy ?accessRights . \ -?record aocat:has_description ?description . \ -?record aocat:has_creator ?contributor . \ -?contributor rdfs:label ?contributorName . \ -?record aocat:has_original_id ?originalId . \ -?record aocat:has_temporal_coverage ?t . \ -?t aocat:has_period ?p . \ -?p skos:prefLabel ?temporalPeriodName . \ -?l skos:prefLabel ?language . \ -BIND("organization" as ?contributorType) . \ -BIND("organization" as ?publisherType) . \ -BIND(uri(concat('http://www.myprefix/location/', md5(concat(str(?spatialLocationLat), str(?spatialLocationLon))))) as ?blocation) \ -} \ -} - - -sparql.query3=PREFIX aocat: \ -PREFIX rdfs: \ -PREFIX skos: \ -PREFIX mine: \ -PREFIX onto: \ -CONSTRUCT {\ - ?record mine:resourcetype "AriadneCatalogEntry" .\ - ?record mine:identifier ?record .\ - ?record mine:publisher ?x .\ - ?x mine:type ?publisherType .\ -?record mine:issued ?issued .\ -?record mine:modified ?modified .\ -?record mine:accessPolicy ?accessPolicy .\ -?record mine:landingPage ?landingPage .\ -?record mine:title ?title .\ -?record mine:accessRights ?accessRights .\ -?record mine:description ?description .\ -?record mine:contributor ?contributor .\ -?record mine:originalId ?originalId .\ -}\ -WHERE {\ -?record aocat:has_access_policy ?accessPolicy .\ -?record aocat:has_landing_page ?z .\ -?z rdfs:label ?landingPage .\ -?record aocat:has_title ?title .\ -?record aocat:has_access_policy ?accessRights .\ -?record aocat:has_description ?description .\ -?record aocat:has_original_id ?originalId .\ -BIND("organization" as ?contributorType) .\ -BIND("organization" as ?publisherType) .\ -}\ -limit 20 - -catalog.entry.path=$[*][?(@['http://www.myprefix/resourcetype'][0]['value']=='AriadneCatalogEntry')] -general.classpath=ariadneplus.elasticsearch.model. -type.path=http://www.myprefix/resourcetype -exclude.predicates=["http://www.myprefix/resourcetype", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] +catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='provided record')] +catalog.entry.collection.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='Dataset Collection')] +general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model. +type.path=https://www.ariadne-infrastructure.eu/property/resourceType +exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] class.map.specifications={\ "AriadneTemporal": {\ "class_type": "prototype",\ "mappings": {\ -"http://www.myprefix/from": {\ +"https://www.ariadne-infrastructure.eu/property/from": {\ "class_field": "From",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/periodName": {\ +"https://www.ariadne-infrastructure.eu/property/periodName": {\ "class_field": "PeriodName",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/until": {\ +"https://www.ariadne-infrastructure.eu/property/until": {\ "class_field": "Until",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/uri": {\ +"https://www.ariadne-infrastructure.eu/property/uri": {\ "class_field": "Uri",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -155,12 +46,12 @@ class.map.specifications={\ "AriadneGeoPoint": {\ "class_type": "unique",\ "mappings": {\ -"http://www.myprefix/lat": {\ +"https://www.ariadne-infrastructure.eu/property/lat": {\ "class_field": "Lat",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/lon": {\ +"https://www.ariadne-infrastructure.eu/property/lon": {\ "class_field": "Lon",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -169,41 +60,51 @@ class.map.specifications={\ },\ "Spatial": {\ "class_type": "prototype",\ -"mappings":{"http://www.myprefix/address": {\ +"mappings":{"https://www.ariadne-infrastructure.eu/property/address": {\ "class_field": "Address",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/coordinateSystem": {\ +"https://www.ariadne-infrastructure.eu/property/coordinateSystem": {\ "class_field": "CoordinateSystem",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/country": {\ +"https://www.ariadne-infrastructure.eu/property/country": {\ "class_field": "Country",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/location": {\ +"https://www.ariadne-infrastructure.eu/property/location": {\ "class_field": "Location",\ "external_reference": "AriadneGeoPoint",\ "substring": "no"\ },\ -"http://www.myprefix/placeName": {\ +"https://www.ariadne-infrastructure.eu/property/lat": {\ +"class_field": "Lat",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lon": {\ +"class_field": "Lon",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/placeName": {\ "class_field": "PlaceName",\ "substring": "no",\ "element_type": "java.lang.String"\ }}\ },\ -"ArcheologicalResourceType": {\ +"ArchaeologicalResourceType": {\ "class_type": "unique",\ "mappings": {\ -"http://www.myprefix/id": {\ +"https://www.ariadne-infrastructure.eu/property/id": {\ "class_field": "Id",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/name": {\ +"https://www.ariadne-infrastructure.eu/property/name": {\ "class_field": "Name",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -213,22 +114,22 @@ class.map.specifications={\ "AgentInfo": {\ "class_type": "prototype",\ "mappings": {\ -"http://www.myprefix/name": {\ +"https://www.ariadne-infrastructure.eu/property/name": {\ "class_field": "Name",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/type": {\ +"https://www.ariadne-infrastructure.eu/property/type": {\ "class_field": "Type",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/email": {\ +"https://www.ariadne-infrastructure.eu/property/email": {\ "class_field": "Email",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/phone": {\ +"https://www.ariadne-infrastructure.eu/property/phone": {\ "class_field": "Phone",\ "substring": "no",\ "element_type": "java.lang.String"\ @@ -238,108 +139,178 @@ class.map.specifications={\ "NativeSubject": {\ "class_type": "prototype",\ "mappings": {\ -"http://www.myprefix/prefLabel": {\ +"https://www.ariadne-infrastructure.eu/property/prefLabel": {\ "class_field": "PrefLabel",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/rdfAbout": {\ +"https://www.ariadne-infrastructure.eu/property/rdfAbout": {\ "class_field": "RdfAbout",\ "substring": "yes",\ "element_type": "java.lang.String"\ }\ }\ },\ +"DerivedSubject": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/prefLabel": {\ +"class_field": "PrefLabel",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/source": {\ +"class_field": "Source",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AatSubject": {\ +"class_type": "prototype",\ +"mappings": {\ +"https://www.ariadne-infrastructure.eu/property/id": {\ +"class_field": "Id",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/label": {\ +"class_field": "Label",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/lang": {\ +"class_field": "Lang",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ "AriadneCatalogEntry": {\ "class_type": "unique",\ "mappings": {\ -"http://www.myprefix/accessPolicy": {\ +"https://www.ariadne-infrastructure.eu/property/accessPolicy": {\ "class_field": "AccessPolicy",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/identifier": {\ +"https://www.ariadne-infrastructure.eu/property/identifier": {\ "class_field": "Identifier",\ "substring": "yes",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/publisher": {\ +"https://www.ariadne-infrastructure.eu/property/publisher": {\ "class_field": "Publisher",\ "external_reference": "AgentInfo",\ "substring": "no"\ },\ -"http://www.myprefix/archeologicalResourceType": {\ -"class_field": "ArcheologicalResourceType",\ -"external_reference": "ArcheologicalResourceType",\ +"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {\ +"class_field": "ArchaeologicalResourceType",\ +"external_reference": "ArchaeologicalResourceType",\ "substring": "no"\ },\ -"http://www.myprefix/issued": {\ +"https://www.ariadne-infrastructure.eu/property/issued": {\ "class_field": "Issued",\ "element_type": "java.lang.String",\ "substring": "no"\ },\ -"http://www.myprefix/resourceType": {\ +"https://www.ariadne-infrastructure.eu/property/resourceType": {\ "class_field": "ResourceType",\ "element_type": "java.lang.String",\ "substring": "no"\ },\ -"http://www.myprefix/modified": {\ +"https://www.ariadne-infrastructure.eu/property/modified": {\ "class_field": "Modified",\ "element_type": "java.lang.String",\ "substring": "no"\ },\ -"http://www.myprefix/nativeSubject": {\ +"https://www.ariadne-infrastructure.eu/property/nativeSubject": {\ "class_field": "NativeSubject",\ "substring": "no",\ "external_reference": "NativeSubject"\ },\ -"http://www.myprefix/spatial": {\ +"https://www.ariadne-infrastructure.eu/property/derivedSubject": {\ +"class_field": "DerivedSubject",\ +"substring": "no",\ +"external_reference": "DerivedSubject"\ +},\ +"https://www.ariadne-infrastructure.eu/property/aatSubjects": {\ +"class_field": "AatSubjects",\ +"substring": "no",\ +"external_reference": "AatSubject"\ +},\ +"https://www.ariadne-infrastructure.eu/property/spatialRegion": {\ "class_field": "Spatial",\ "substring": "no",\ "external_reference": "Spatial"\ },\ -"http://www.myprefix/landingPage": {\ +"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": {\ +"class_field": "Spatial",\ +"substring": "no",\ +"external_reference": "Spatial"\ +},\ +"https://www.ariadne-infrastructure.eu/property/landingPage": {\ "class_field": "LandingPage",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/title": {\ +"https://www.ariadne-infrastructure.eu/property/placeName": {\ +"class_field": "PlaceName",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"https://www.ariadne-infrastructure.eu/property/title": {\ "class_field": "Title",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/accessRights": {\ +"https://www.ariadne-infrastructure.eu/property/accessRights": {\ "class_field": "AccessRights",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/description": {\ +"https://www.ariadne-infrastructure.eu/property/description": {\ "class_field": "Description",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/contributor": {\ +"https://www.ariadne-infrastructure.eu/property/creator": {\ +"class_field": "Creator",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/contributor": {\ "class_field": "Contributor",\ "substring": "no",\ "external_reference": "AgentInfo"\ },\ -"http://www.myprefix/originalId": {\ +"https://www.ariadne-infrastructure.eu/property/legalResponsible": {\ +"class_field": "LegalResponsible",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/owner": {\ +"class_field": "Owner",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"https://www.ariadne-infrastructure.eu/property/originalId": {\ "class_field": "OriginalId",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/temporal": {\ +"https://www.ariadne-infrastructure.eu/property/uri": {\ "class_field": "Temporal",\ "substring": "no",\ "external_reference": "AriadneTemporal"\ },\ -"http://www.myprefix/language": {\ +"https://www.ariadne-infrastructure.eu/property/language": {\ "class_field": "Language",\ "substring": "no",\ "element_type": "java.lang.String"\ },\ -"http://www.myprefix/partOf": {\ -"class_field": "PartOf",\ +"https://www.ariadne-infrastructure.eu/property/partOf": {\ +"class_field": "IsPartOf",\ "substring": "no",\ "element_type": "java.lang.String"\ }\ diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql new file mode 100644 index 0000000..40d936f --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql @@ -0,0 +1,44 @@ +PREFIX aocat: +PREFIX skos: +PREFIX time: +PREFIX ariadneplus: +INSERT { + GRAPH { + ?record aocat:was_issued ?issued . + ?record aocat:was_modified ?modified . + ?record aocat:has_contributor ?contributor . + ?record aocat:has_responsible ?legalResponsible . + ?record aocat:has_owner ?owner . + ?record aocat:has_publisher ?publisher . + ?record aocat:has_access_rights ?accessRights . + ?record aocat:has_ARIADNE_subject ?archeologicalResourceType . + } +} +WHERE { + ?record aocat:is_part_of ?collection . + ?collection aocat:was_issued ?issued . + ?collection aocat:was_modified ?modified . + ?collection aocat:has_contributor ?contributor . + ?contributor aocat:has_name ?contributorName . + ?collection aocat:has_responsible ?legalResponsible . + ?legalResponsible aocat:has_name ?legalResponsibleName . + ?collection aocat:has_owner ?owner . + ?owner aocat:has_name ?ownerName . + ?collection aocat:has_publisher ?publisher . + ?publisher aocat:has_name ?publisherName . + ?collection aocat:has_access_rights ?accessRights . + ?collection aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + ?contributor aocat:has_email ?contributorEmail . + } + optional { + ?legalResponsible aocat:has_email ?legalResponsibleEmail . + } + optional { + ?owner aocat:has_email ?ownerEmail . + } + optional { + ?publisher aocat:has_email ?publisherEmail . + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_aatplus_graph.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_aatplus_graph.sparql new file mode 100644 index 0000000..8565b35 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_aatplus_graph.sparql @@ -0,0 +1,33 @@ +PREFIX aocat: +PREFIX skos: +PREFIX time: +PREFIX ariadneplus: +PREFIX rdf: +INSERT { + GRAPH { + ?record aocat:has_derived_subject ?aat . + } +} +USING +USING +WHERE { + { + ?record aocat:has_native_subject ?native_subject . + ?native_subject skos:exactMatch ?aat . + } + union + { + ?record aocat:has_native_subject ?native_subject . + ?native_subject skos:broadMatch ?aat . + } + union + { + ?record aocat:has_native_subject ?native_subject . + ?native_subject skos:closeMatch ?aat . + } + union + { + ?record aocat:has_native_subject ?native_subject . + ?native_subject skos:narrowMatch ?aat . + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_periodoplus_graph.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_periodoplus_graph.sparql new file mode 100644 index 0000000..77284a8 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/gen_periodoplus_graph.sparql @@ -0,0 +1,23 @@ +PREFIX aocat: +PREFIX skos: +PREFIX time: +PREFIX ariadneplus: +INSERT { + GRAPH { + ?temporal aocat:has_period ?periodO . + ?temporal aocat:from ?temporalFrom . + ?temporal aocat:until ?temporalUntil . + } +} +WHERE { + ?temporal aocat:has_native_period ?native_period . + ?native_period skos:prefLabel ?native_label . + optional { + ?periodO skos:altLabel ?native_label . + ?periodO skos:inScheme . + ?periodO time:intervalStartedBy ?intervalStartedBy . + ?intervalStartedBy skos:prefLabel ?temporalFrom . + ?periodO time:intervalFinishedBy ?intervalFinishedBy . + ?intervalFinishedBy skos:prefLabel ?temporalUntil . + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql new file mode 100644 index 0000000..33449aa --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql @@ -0,0 +1,124 @@ +PREFIX aocat: +PREFIX rdfs: +PREFIX skos: +PREFIX aoprop: +PREFIX onto: +PREFIX ariadneplus: +PREFIX time: + +CONSTRUCT { +%record aoprop:identifier %record . +%record aoprop:originalId ?originalId . +%record aoprop:issued ?issued . +%record aoprop:modified ?modified . +%record aoprop:partOf ?partOf . +%record aoprop:creator ?creator . +?creator aoprop:name ?creatorName . +?creator aoprop:email ?creatorEmail . +%record aoprop:contributor ?contributor . +?contributor aoprop:name ?contributorName . +?contributor aoprop:email ?contributorEmail . +%record aoprop:legalResponsible ?legalResponsible . +?legalResponsible aoprop:name ?legalResponsibleName . +?legalResponsible aoprop:email ?legalResponsibleEmail . +%record aoprop:owner ?owner . +?owner aoprop:name ?ownerName . +?owner aoprop:email ?ownerEmail . +%record aoprop:publisher ?publisher . +?publisher aoprop:name ?publisherName . +?publisher aoprop:email ?publisherEmail . +%record aoprop:accessPolicy ?accessPolicy . +%record aoprop:accessRights ?accessRights . +%record aoprop:landingPage ?landingPage . +%record aoprop:spatialRegion ?spatialRegion . +?spatialRegion aoprop:placeName ?spatialPlaceName . +?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . +%record aoprop:spatialRegionPoint ?spatialRegionPoint . +?spatialRegionPoint aoprop:lat ?spatialLocationLat . +?spatialRegionPoint aoprop:lon ?spatialLocationLon . +%record aoprop:spatialRegionBox ?spatialRegionBox . +?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat . +?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon . +?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat . +?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon . +%record aoprop:uri ?temporal . +?temporal aoprop:periodName ?temporalPeriodName . +?temporal aoprop:from ?temporalFrom . +?temporal aoprop:until ?temporalUntil . +%record aoprop:uri ?temporalNative . +?temporalNative aoprop:periodName ?temporalNativePeriodName . +%record aoprop:archeologicalResourceType ?archeologicalResourceType . +?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . +%record aoprop:resourceType ?resourceType . +%record aoprop:nativeSubject ?nativeSubject . +?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . +?nativeSubject aoprop:rdfAbout ?nativeSubject . +%record aoprop:derivedSubject ?derivedSubject . +?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel . +?derivedSubject aoprop:source "Getty AAT" . +%record aoprop:aatSubjects ?derivedSubject . +?derivedSubject aoprop:id ?derivedSubject . +?derivedSubject aoprop:label ?derivedSubjectPrefLabel . +?derivedSubject aoprop:lang ?aatSubjectsLang . +%record aoprop:title ?title . +%record aoprop:description ?description . +%record aoprop:language ?language . +} +from +from +from +from +from +where { + %record aocat:has_language / skos:prefLabel ?language . + %record aocat:has_original_id ?originalId . + %record aocat:has_creator ?creator . + ?creator aocat:has_name ?creatorName . + %record aocat:has_title ?title . + %record aocat:has_type / skos:prefLabel ?resourceType . + optional { + ?creator aocat:has_email ?creatorEmail . + } + optional { + %record aocat:has_description ?description . + } + optional { + %record aocat:has_access_policy / rdfs:label ?accessPolicy . + } + optional { + %record aocat:has_landing_page / rdfs:label ?landingPage . + } + + { + select * + where { + %record aocat:was_issued ?issued . + %record aocat:was_modified ?modified . + %record aocat:has_contributor ?contributor . + ?contributor aocat:has_name ?contributorName . + %record aocat:has_responsible ?legalResponsible . + ?legalResponsible aocat:has_name ?legalResponsibleName . + %record aocat:has_owner ?owner . + ?owner aocat:has_name ?ownerName . + %record aocat:has_publisher ?publisher . + ?publisher aocat:has_name ?publisherName . + %record aocat:has_access_rights ?accessRights . + %record aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + ?contributor aocat:has_email ?contributorEmail . + } + optional { + ?legalResponsible aocat:has_email ?legalResponsibleEmail . + } + optional { + ?owner aocat:has_email ?ownerEmail . + } + optional { + ?publisher aocat:has_email ?publisherEmail . + } + } + } + + BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang) +} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql similarity index 75% rename from dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt rename to dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql index 268fafe..84b4730 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/query.txt +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data.sparql @@ -45,22 +45,31 @@ CONSTRUCT { ?temporal aoprop:periodName ?temporalPeriodName . ?temporal aoprop:from ?temporalFrom . ?temporal aoprop:until ?temporalUntil . +?record aoprop:uri ?temporalNative . +?temporalNative aoprop:periodName ?temporalNativePeriodName . ?record aoprop:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . ?record aoprop:resourceType ?resourceType . ?record aoprop:nativeSubject ?nativeSubject . ?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . ?nativeSubject aoprop:rdfAbout ?nativeSubject . -?nativeSubject aoprop:aatSubjectsId ?aatSubjectsId . +?record aoprop:derivedSubject ?derivedSubject . +?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel . +?derivedSubject aoprop:source "Getty AAT" . +?record aoprop:aatSubjects ?derivedSubject . +?derivedSubject aoprop:id ?derivedSubject . +?derivedSubject aoprop:label ?derivedSubjectPrefLabel . +?derivedSubject aoprop:lang ?aatSubjectsLang . ?record aoprop:title ?title . ?record aoprop:description ?description . ?record aoprop:language ?language . } from -from -from -from -where { +from +from +from +from +where { ?record aocat:has_language / skos:prefLabel ?language . ?record aocat:has_original_id ?originalId . ?record aocat:is_part_of ?partOf . @@ -68,19 +77,15 @@ where { ?creator aocat:has_name ?creatorName . ?record aocat:has_title ?title . ?record aocat:has_type / skos:prefLabel ?resourceType . + ?record aocat:has_native_subject ?nativeSubject . + ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . + optional { + ?record aocat:has_derived_subject ?derivedSubject . + ?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel . + } optional { ?creator aocat:has_email ?creatorEmail . } - optional { - ?record aocat:has_native_subject ?nativeSubject . - ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . - optional { - ?nativeSubject skos:exactMatch ?aatSubjectsId . - } - optional { - ?nativeSubject skos:closeMatch ?aatSubjectsId . - } - } optional { ?record aocat:has_description ?description . } @@ -90,38 +95,35 @@ where { optional { ?record aocat:has_landing_page / rdfs:label ?landingPage . } - optional { - ?record aocat:has_temporal_coverage ?temporal . - ?temporal aocat:has_period / rdfs:label ?temporalPeriodName . + optional { + ?record aocat:has_temporal_coverage ?temporalNative . + ?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName . } optional { ?record aocat:has_temporal_coverage ?temporal . - ?temporal aocat:has_native_period ?nativePeriod . - ?nativePeriod skos:prefLabel ?temporalPeriodName . + ?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName . optional { - ?nativePeriod skos:exactMatch ?periodORef . - ?periodORef time:intervalStartedBy ?intervalStartedBy . - ?intervalStartedBy skos:prefLabel ?temporalFrom . - ?periodORef time:intervalFinishedBy ?intervalFinishedBy . - ?intervalFinishedBy skos:prefLabel ?temporalUntil . - } + ?temporal aocat:from ?temporalFrom . + ?temporal aocat:until ?temporalUntil . + } } + { select * where { ?record aocat:is_part_of ?collection . - ?collection aocat:was_issued ?issued . - ?collection aocat:was_modified ?modified . - ?collection aocat:has_contributor ?contributor . + ?record aocat:was_issued ?issued . + ?record aocat:was_modified ?modified . + ?record aocat:has_contributor ?contributor . ?contributor aocat:has_name ?contributorName . - ?collection aocat:has_responsible ?legalResponsible . + ?record aocat:has_responsible ?legalResponsible . ?legalResponsible aocat:has_name ?legalResponsibleName . - ?collection aocat:has_owner ?owner . + ?record aocat:has_owner ?owner . ?owner aocat:has_name ?ownerName . - ?collection aocat:has_publisher ?publisher . + ?record aocat:has_publisher ?publisher . ?publisher aocat:has_name ?publisherName . - ?collection aocat:has_access_rights ?accessRights . - ?collection aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?record aocat:has_access_rights ?accessRights . + ?record aocat:has_ARIADNE_subject ?archeologicalResourceType . ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . optional { ?contributor aocat:has_email ?contributorEmail . @@ -137,6 +139,7 @@ where { } } } + { select * where { @@ -159,5 +162,5 @@ where { } } } - + BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang) } \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql new file mode 100644 index 0000000..798e5cb --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql @@ -0,0 +1,166 @@ +PREFIX aocat: +PREFIX rdfs: +PREFIX skos: +PREFIX aoprop: +PREFIX onto: +PREFIX ariadneplus: +PREFIX time: + +CONSTRUCT { +%record aoprop:identifier %record . +%record aoprop:originalId ?originalId . +%record aoprop:issued ?issued . +%record aoprop:modified ?modified . +%record aoprop:partOf ?partOf . +%record aoprop:creator ?creator . +?creator aoprop:name ?creatorName . +?creator aoprop:email ?creatorEmail . +%record aoprop:contributor ?contributor . +?contributor aoprop:name ?contributorName . +?contributor aoprop:email ?contributorEmail . +%record aoprop:legalResponsible ?legalResponsible . +?legalResponsible aoprop:name ?legalResponsibleName . +?legalResponsible aoprop:email ?legalResponsibleEmail . +%record aoprop:owner ?owner . +?owner aoprop:name ?ownerName . +?owner aoprop:email ?ownerEmail . +%record aoprop:publisher ?publisher . +?publisher aoprop:name ?publisherName . +?publisher aoprop:email ?publisherEmail . +%record aoprop:accessPolicy ?accessPolicy . +%record aoprop:accessRights ?accessRights . +%record aoprop:landingPage ?landingPage . +%record aoprop:spatialRegion ?spatialRegion . +?spatialRegion aoprop:placeName ?spatialPlaceName . +?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . +%record aoprop:spatialRegionPoint ?spatialRegionPoint . +?spatialRegionPoint aoprop:lat ?spatialLocationLat . +?spatialRegionPoint aoprop:lon ?spatialLocationLon . +%record aoprop:spatialRegionBox ?spatialRegionBox . +?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat . +?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon . +?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat . +?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon . +%record aoprop:uri ?temporal . +?temporal aoprop:periodName ?temporalPeriodName . +?temporal aoprop:from ?temporalFrom . +?temporal aoprop:until ?temporalUntil . +%record aoprop:uri ?temporalNative . +?temporalNative aoprop:periodName ?temporalNativePeriodName . +%record aoprop:archeologicalResourceType ?archeologicalResourceType . +?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . +%record aoprop:resourceType ?resourceType . +%record aoprop:nativeSubject ?nativeSubject . +?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel . +?nativeSubject aoprop:rdfAbout ?nativeSubject . +%record aoprop:derivedSubject ?derivedSubject . +?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel . +?derivedSubject aoprop:source "Getty AAT" . +%record aoprop:aatSubjects ?derivedSubject . +?derivedSubject aoprop:id ?derivedSubject . +?derivedSubject aoprop:label ?derivedSubjectPrefLabel . +?derivedSubject aoprop:lang ?aatSubjectsLang . +%record aoprop:title ?title . +%record aoprop:description ?description . +%record aoprop:language ?language . +} +from +from +from +from +from +where { + %record aocat:has_language / skos:prefLabel ?language . + %record aocat:has_original_id ?originalId . + %record aocat:is_part_of ?partOf . + %record aocat:has_creator ?creator . + ?creator aocat:has_name ?creatorName . + %record aocat:has_title ?title . + %record aocat:has_type / skos:prefLabel ?resourceType . + %record aocat:has_native_subject ?nativeSubject . + ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel . + optional { + %record aocat:has_derived_subject ?derivedSubject . + ?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel . + } + optional { + ?creator aocat:has_email ?creatorEmail . + } + optional { + %record aocat:has_description ?description . + } + optional { + %record aocat:has_access_policy / rdfs:label ?accessPolicy . + } + optional { + %record aocat:has_landing_page / rdfs:label ?landingPage . + } + optional { + %record aocat:has_temporal_coverage ?temporalNative . + ?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName . + } + optional { + %record aocat:has_temporal_coverage ?temporal . + ?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName . + optional { + ?temporal aocat:from ?temporalFrom . + ?temporal aocat:until ?temporalUntil . + } + } + + { + select * + where { + %record aocat:is_part_of ?collection . + %record aocat:was_issued ?issued . + %record aocat:was_modified ?modified . + %record aocat:has_contributor ?contributor . + ?contributor aocat:has_name ?contributorName . + %record aocat:has_responsible ?legalResponsible . + ?legalResponsible aocat:has_name ?legalResponsibleName . + %record aocat:has_owner ?owner . + ?owner aocat:has_name ?ownerName . + %record aocat:has_publisher ?publisher . + ?publisher aocat:has_name ?publisherName . + %record aocat:has_access_rights ?accessRights . + %record aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + optional { + ?contributor aocat:has_email ?contributorEmail . + } + optional { + ?legalResponsible aocat:has_email ?legalResponsibleEmail . + } + optional { + ?owner aocat:has_email ?ownerEmail . + } + optional { + ?publisher aocat:has_email ?publisherEmail . + } + } + } + + { + select * + where { + %record aocat:has_spatial_coverage ?spatialRegion . + ?spatialRegion aocat:has_place_name ?spatialPlaceName . + optional { + ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem . + } + optional { + %record aocat:has_spatial_coverage ?spatialRegionPoint . + ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; + aocat:has_longitude ?spatialLocationLon . + } + optional { + %record aocat:has_spatial_coverage ?spatialRegionBox . + ?spatialRegionBox aocat:has_bounding_box_max_lat ?spatialLocationBBMaxLat ; + aocat:has_bounding_box_max_lon ?spatialLocationBBMaxLon ; + aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ; + aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ; + } + } + } + BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang) +} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/D14_Software.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/D14_Software.sparql deleted file mode 100644 index f8fa1bd..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/D14_Software.sparql +++ /dev/null @@ -1,106 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - - <${subjectURL}> crm:P102_has_title ?title . - <${subjectURL}> crm:P3_has_note ?description . - <${subjectURL}> a ?type . - <${subjectURL}> rdfs:label ?label . - - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - - <${subjectURL}> crm:P2_has_type ?e55TypeLabel . - - <${subjectURL}> crmpe:PP7i_is_software_object_hosted_by ?host . - <${subjectURL}> crmpe:PP32i_is_curated_by ?curatingService . - - <${subjectURL}> crmpe:PP17_has_snapshot ?snapshot1 . - <${subjectURL}> crmpe:PP17i_is_snapshot_of ?snapshot2 . - - <${subjectURL}> crmpe:PP21_has_software_part ?sw1 . - <${subjectURL}> crmpe:PP21i_is_software_part_of ?sw2 . - <${subjectURL}> crmpe:PP22_has_release ?sw3 . - <${subjectURL}> crmpe:PP22i_is_release_of ?sw4 . - - <${subjectURL}> ?adopter . - <${subjectURL}> ?creationtime . -} -WHERE { - <${subjectURL}> a crmdig:D14_Software . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} -UNION { - <${subjectURL}> crm:P2_has_type ?e55Type . - ?e55Type rdfs:label ?e55TypeLabel . -} - UNION{ - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - } - UNION{<${subjectURL}> crmpe:PP7i_is_software_object_hosted_by ?host . } - UNION{?host crmpe:PP7_hosts_software_object <${subjectURL}> . } - UNION{<${subjectURL}> crmpe:PP32i_is_curated_by ?curatingService .} - UNION{?curatingService crmpe:PP32_curates <${subjectURL}> .} - UNION{ - <${subjectURL}> crmpe:PP32i_is_curated_by ?curatingService . - ?curatingService crmpe:PP31_uses_curation_plan ?curationplan . - } -UNION{ - ?curatingService crmpe:PP32_curates <${subjectURL}> . - ?curatingService crmpe:PP31_uses_curation_plan ?curationplan . -} - UNION{<${subjectURL}> crmpe:PP17_has_snapshot ?snapshot1 .} - UNION{ ?snapshot1 crmpe:PP17i_is_snapshot_of <${subjectURL}> .} - UNION{<${subjectURL}> crmpe:PP17i_is_snapshot_of ?snapshot2 .} - UNION{ ?snapshot2 crmpe:PP17_has_snapshot <${subjectURL}> .} - - UNION{<${subjectURL}> crmpe:PP21_has_software_part ?sw1 .} - UNION{?sw1 crmpe:PP21i_is_software_part_of <${subjectURL}>.} - - UNION{ ?sw2 crmpe:PP21_has_software_part <${subjectURL}> .} - UNION{<${subjectURL}> crmpe:PP21i_is_software_part_of ?sw2 .} - - UNION{<${subjectURL}> crmpe:PP22_has_release ?sw3 .} - UNION{?sw3 crmpe:PP22i_is_release_of <${subjectURL}> .} - - UNION{<${subjectURL}> crmpe:PP22i_is_release_of ?sw4 .} - UNION{?sw4 crmpe:PP22_has_release <${subjectURL}> .} - - UNION{ - ?adopter crm:P94i_was_created_by ?event . - ?event crmdig:L23_used_software_or_firmware <${subjectURL}> . - } - UNION { - <${subjectURL}> crmdig:L23i_was_software_or_firmware_used_by ?event . - ?adopter crm:P94i_was_created_by ?event . - } - UNION{ - <${subjectURL}> crm:P94i_was_created_by ?event1 . - ?event1 crm:P2_has_type . - ?event1 crm:P4_has_time-span ?timespan . - ?timespan crm:P81_ongoing_throughout ?creationtime . - } -} -LIMIT 1000 \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E29_Design_or_Procedure.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E29_Design_or_Procedure.sparql deleted file mode 100644 index 5ef1d16..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E29_Design_or_Procedure.sparql +++ /dev/null @@ -1,51 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { -<${subjectURL}> crm:P1_is_identified_by ?IDRes . -?IDRes a crm:E42_Identifier . -?IDRes rdfs:label ?ID_label . - -<${subjectURL}> crm:P102_has_title ?title . -<${subjectURL}> crm:P3_has_note ?description . -<${subjectURL}> a ?type . -<${subjectURL}> rdfs:label ?label . - -<${subjectURL}> crm:P2_has_type ?e55TypeLabel . - -<${subjectURL}> ?adopter . -<${subjectURL}> ?creationtime . -} -WHERE { - <${subjectURL}> a crm:E29_Design_or_Procedure . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} -UNION { - <${subjectURL}> crm:P2_has_type ?e55Type . - ?e55Type rdfs:label ?e55TypeLabel . -} -UNION { - ?adopter crmpe:PP31_uses_curation_plan <${subjectURL}> . -} -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?event1 . - ?event1 crm:P2_has_type . - ?event1 crm:P4_has_time-span ?timespan . - ?timespan crm:P81_ongoing_throughout ?creationtime . -} -} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E39_Actor.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E39_Actor.sparql deleted file mode 100644 index dfbad19..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E39_Actor.sparql +++ /dev/null @@ -1,68 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { -<${subjectURL}> rdfs:label ?label . -<${subjectURL}> a ?type . -<${subjectURL}> crm:P2_has_type ?e55TypeLabel . - -<${subjectURL}> crm:P1_is_identified_by ?IDRes . -?IDRes a crm:E42_Identifier . -?IDRes rdfs:label ?ID_label . - -<${subjectURL}> crm:P102_has_title ?title . -<${subjectURL}> crm:P3_has_note ?description . - -<${subjectURL}> crm:P76_has_contact_point ?contactpoint . -?contactpoint crm:P2_has_type ?contactpointType . -?contactpoint rdfs:label ?contactpointLabel . -?contactpointType rdfs:label ?contactpointTypeLabel . - -<${subjectURL}> crm:P107_has_current_or_former_member ?member1 . -<${subjectURL}> crm:P107i_is_current_or_former_member_of ?team . -<${subjectURL}> crmpe:PP2i_provides ?service . - -<${subjectURL}> crmpe:PP44i_is_maintaining_team_of ?maintained . -} -WHERE { - <${subjectURL}> a crm:E39_Actor . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ <${subjectURL}> crm:P3_has_note ?description .} -UNION{ <${subjectURL}> rdfs:label ?label .} -UNION { - <${subjectURL}> crm:P2_has_type ?e55Type . - ?e55Type rdfs:label ?e55TypeLabel . -} -UNION{ - <${subjectURL}> crm:P76_has_contact_point ?contactpoint . -} - UNION{ - <${subjectURL}> crm:P76_has_contact_point ?contactpoint . - ?contactpoint rdfs:label ?contactpointLabel . -} -UNION{ - <${subjectURL}> crm:P76_has_contact_point ?contactpoint . - ?contactpoint crm:P2_has_type ?contactpointType . - ?contactpointType rdfs:label ?contactpointTypeLabel . - } - UNION{ <${subjectURL}> crm:P107_has_current_or_former_member ?member1 .} - UNION{ ?member1 crm:P107_has_current_or_former_member <${subjectURL}> .} - UNION{ <${subjectURL}> crm:P107i_is_current_or_former_member_of ?team .} - UNION{ ?team crm:P107_has_current_or_former_member <${subjectURL}> .} - - UNION{ <${subjectURL}> crmpe:PP2i_provides ?service .} - UNION{ ?service crmpe:PP2_provided_by <${subjectURL}> .} - - UNION{ <${subjectURL}> crmpe:PP44i_is_maintaining_team_of ?maintained .} - UNION{ ?maintained crmpe:PP44_has_maintaining_team <${subjectURL}> .} -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E78_Collection.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E78_Collection.sparql deleted file mode 100644 index 98dd6ae..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/E78_Collection.sparql +++ /dev/null @@ -1,73 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - - <${subjectURL}> crm:P102_has_title ?title . - <${subjectURL}> crm:P3_has_note ?description . - <${subjectURL}> a ?type . - <${subjectURL}> rdfs:label ?label . - - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - - <${subjectURL}> crm:P2_has_type ?e55TypeLabel . - - <${subjectURL}> crm:P46_is_composed_of ?Col_Part . - - <${subjectURL}> crmpe:PP4i_is_object_hosted_by ?host_serv . - <${subjectURL}> crmpe:PP32i_is_curated_by ?curator . - <${subjectURL}> ?creator . - - <${subjectURL}> crm:P72_has_language ?languageLabel . -} -WHERE { - <${subjectURL}> a crm:E78_Collection . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} -UNION{ -<${subjectURL}> crm:P129_is_about ?about . -?about a ?aboutType . -?about rdfs:label ?aboutLabel . -} -UNION { -<${subjectURL}> crm:P2_has_type ?e55Type . -?e55Type rdfs:label ?e55TypeLabel . -} -UNION{ <${subjectURL}> crm:P46_is_composed_of ?Col_Part .} -UNION{ ?Col_Part crm:P46i_forms_part_of <${subjectURL}> .} - -UNION{ <${subjectURL}> crmpe:PP4i_is_object_hosted_by ?host_serv .} -UNION{ <${subjectURL}> crmpe:PP32i_is_curated_by ?curator .} -UNION{ ?host_serv crmpe:PP4_hosts_object <${subjectURL}> .} -UNION{ ?curator crmpe:PP32_curates <${subjectURL}> .} - -UNION { - <${subjectURL}> crm:P94i_was_created_by ?Creation_event . - ?Creation_event crm:P14_carried_out_by ?creator . -} -UNION { - <${subjectURL}> crm:P72_has_language ?language . - ?language rdfs:label ?languageLabel . -} - -} LIMIT 30000 \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE18_Dataset.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE18_Dataset.sparql deleted file mode 100644 index 1012e27..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE18_Dataset.sparql +++ /dev/null @@ -1,138 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - - <${subjectURL}> crm:P102_has_title ?title . - <${subjectURL}> crm:P3_has_note ?description . - <${subjectURL}> a ?type . - <${subjectURL}> rdfs:label ?label . - - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - - <${subjectURL}> crm:P2_has_type ?e55TypeLabel . - - <${subjectURL}> crmpe:PP8i_is_dataset_hosted_by ?host . - <${subjectURL}> crmpe:PP32i_is_curated_by ?curator . - - <${subjectURL}> crm:P106i_forms_part_of ?container . - <${subjectURL}> crm:P106_is_composed_of ?d3 . - - <${subjectURL}> crmpe:PP39_is_metadata_for ?anotherdataset . - <${subjectURL}> crmpe:PP39i_has_metadata ?metadata . - - <${subjectURL}> crmpe:PP17i_is_snapshot_of ?snapshotted . - <${subjectURL}> crmpe:PP17_has_snapshot ?snapshot . - - <${subjectURL}> ?encoding_label . - <${subjectURL}> ?sw . - <${subjectURL}> ?creator . - <${subjectURL}> ?creationtime . - <${subjectURL}> ?updatetime . - <${subjectURL}> ?custodytime . - <${subjectURL}> ?custodialhistoryNote . - <${subjectURL}> ?curationtime . - <${subjectURL}> crm:P72_has_language ?languageLabel . -} -WHERE { - <${subjectURL}> a crmpe:PE18_Dataset . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} - UNION{ - <${subjectURL}> crm:P129_is_about ?about . - ?about a ?aboutType . - ?about rdfs:label ?aboutLabel . - } - UNION { - <${subjectURL}> crm:P2_has_type ?e55Type . - ?e55Type rdfs:label ?e55TypeLabel . - } - UNION{ <${subjectURL}> crmpe:PP8i_is_dataset_hosted_by ?host . } - UNION{ ?host crmpe:PP8_hosts_dataset <${subjectURL}> . } - UNION{ <${subjectURL}> crmpe:PP32i_is_curated_by ?curator .} - UNION{ ?curator crmpe:PP32_curates <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP32i_is_curated_by ?curator .} - UNION{ ?curator crmpe:PP32_curates <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP31_uses_curation_plan ?curationplan .} - UNION{ <${subjectURL}> crm:P106i_forms_part_of ?container .} - UNION{ ?container crm:P106_is_composed_of <${subjectURL}> .} - UNION{ <${subjectURL}> crm:P106_is_composed_of ?d3 .} - UNION{ ?d3 crm:P106i_forms_part_of <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP39_is_metadata_for ?anotherdataset .} - UNION{ ?anotherdataset crmpe:PP39i_has_metadata <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP39i_has_metadata ?metadata .} - UNION{ ?metadata crmpe:PP39_is_metadata_for <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP17_has_snapshot ?snapshot .} - UNION{ ?snapshot crmpe:PP17i_is_snapshot_of <${subjectURL}> .} - UNION{ <${subjectURL}> crmpe:PP17i_is_snapshot_of ?snapshotted .} - UNION{ ?snapshotted crmpe:PP17_has_snapshot <${subjectURL}> .} - UNION{ - <${subjectURL}> crm:P94i_was_created_by ?Creation_event . - ?Creation_event crm:P33_used_specific_technique ?technique . - ?technique crm:P2_has_type ?techniquetype . - ?techniquetype rdfs:label ?encoding_label . - } -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?Creation_event . - ?Creation_event crm:P2_has_type . - ?Creation_event crmdig:L23_used_software_or_firmware ?sw . -} -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?Creation_event . - ?Creation_event crm:P2_has_type . - ?Creation_event crm:P14_carried_out_by ?creator . - } -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?event1 . - ?event1 crm:P2_has_type . - ?event1 crm:P4_has_time-span ?timespan . - ?timespan crm:P81_ongoing_throughout ?creationtime . - } -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?updateEvent . - ?updateEvent crm:P2_has_type . - ?updateEvent crm:P4_has_time-span ?timespanUpdate . - ?timespanUpdate crm:P82_at_some_time_within ?updatetime . -} -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?custodialhistory . - ?custodialhistory crm:P2_has_type . - ?custodialhistory crm:P3_has_note ?custodialhistoryNote . -} -UNION{ - <${subjectURL}> crm:P94i_was_created_by ?custodialhistory . - ?custodialhistory crm:P2_has_type . - ?custodialhistory crm:P4_has_time-span ?timespanCustody . - ?timespanCustody crm:P82_at_some_time_within ?custodytime . -} -UNION { - <${subjectURL}> ?hasCurationActivity ?curationActivity . - ?curationActivity a crm:E87_Curation_Activity . - ?curationActivity crm:P4_has_time-span ?curationActivitytimespan . - ?curationActivitytimespan crm:P82_at_some_time_within ?curationtime . -} -UNION { - <${subjectURL}> crm:P72_has_language ?language . - ?language rdfs:label ?languageLabel . -} -} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE1_Service.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE1_Service.sparql deleted file mode 100644 index 873ea06..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE1_Service.sparql +++ /dev/null @@ -1,148 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { -<${subjectURL}> crm:P1_is_identified_by ?IDRes . -?IDRes a crm:E42_Identifier . -?IDRes rdfs:label ?ID_label . - -<${subjectURL}> crm:P102_has_title ?title . -<${subjectURL}> crm:P3_has_note ?description . -<${subjectURL}> rdfs:label ?label . -<${subjectURL}> a ?type . -<${subjectURL}> crm:P2_has_type ?atypeLabel . - -<${subjectURL}> crmpe:PP2_provided_by ?provider . -?provider crm:P1_is_identified_by ?providerTitle . -?provider rdfs:label ?providerLabel . -?provider crm:P3_has_note ?providerDescr . -?provider crm:P76_has_contact_point ?contactpoint . -?contactpoint crm:P2_has_type ?contactpointType . -?contactpoint rdfs:label ?contactpointLabel . -?contactpointType rdfs:label ?contactpointTypeLabel . - -<${subjectURL}> crmpe:PP4_hosts_object ?object . - -<${subjectURL}> crmpe:PP28_has_designated_access_point ?accesspoint . -<${subjectURL}> crmpe:PP1i_is_currently_offered_by ?offerer . -<${subjectURL}> crmpe:PP45_has_competency ?competenceLabel . - -<${subjectURL}> crmpe:PP31_uses_curation_plan ?curationplan . - -<${subjectURL}> crmpe:PP29_uses_access_protocol ?protocol . -?protocol rdfs:label ?protocolLabel . - -<${subjectURL}> crmpe:PP32_curates ?object2 . - -<${subjectURL}> crmpe:PP15_delivers_on_request ?delivered . -<${subjectURL}> crmpe:PP14_runs_on_request ?ran . - -<${subjectURL}> crmpe:PP42_has_declarative_time ?declTime . - -<${subjectURL}> ?availabilityLabel . -<${subjectURL}> ?activitytypeLabel . - - <${subjectURL}> crm:P16_used_specific_object ?Conditions_of_Use . - ?Conditions_of_Use a crm:E30_Right . - ?Conditions_of_Use crm:P3_has_note ?Conditions_of_Use_Note . - ?Conditions_of_Use crm:P2_has_type ?Conditions_of_Use_Type . - ?Conditions_of_Use_Type rdfs:label ?Conditions_of_Use_Type_label . - -} -WHERE { - <${subjectURL}> a crmpe:PE1_Service . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION -{ - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . - FILTER NOT EXISTS { ?titleRes a crm:E51_Contact_Point } -} -UNION{ - <${subjectURL}> crm:P3_has_note ?description . -} -UNION{ - <${subjectURL}> rdfs:label ?label . -} - UNION{ - <${subjectURL}> crm:P2_has_type ?atype . - ?atype rdfs:label ?atypeLabel . -} -UNION{ - <${subjectURL}> crm:P2_has_type ?availability . - ?availability rdfs:label ?availabilityLabel . - FILTER(STRSTARTS(STR(?availability), "http://ariadneplus.d4science.org/handle/AriadnePlus/REG/Concept/availabilitytype")) -} -UNION{ - <${subjectURL}> crm:P2_has_type ?activitytype . - ?activitytype rdfs:label ?activitytypeLabel . - FILTER(STRSTARTS(STR(?activitytype), "http://ariadneplus.d4science.org/handle/AriadnePlus/REG/Concept/activitytype")) -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider crm:P1_is_identified_by ?providerTitleRes . - ?providerTitleRes rdfs:label ?providerTitle . -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider crm:P3_has_note ?providerDescr . -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider rdfs:label ?providerLabel . -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider crm:P76_has_contact_point ?contactpoint . -?contactpoint rdfs:label ?contactpointLabel . -} -UNION { -<${subjectURL}> crmpe:PP2_provided_by ?provider . - ?provider crm:P76_has_contact_point ?contactpoint . - ?contactpoint crm:P2_has_type ?contactpointType . - ?contactpointType rdfs:label ?contactpointTypeLabel .} -UNION { <${subjectURL}> crmpe:PP28_has_designated_access_point ?accesspoint .} -UNION { ?accesspoint crmpe:PP28i_is_designated_access_point_of <${subjectURL}> .} -UNION { <${subjectURL}> crmpe:PP1i_is_currently_offered_by ?offerer .} -UNION { ?offerer crmpe:PP1_currently_offers <${subjectURL}> .} -UNION { - <${subjectURL}> crmpe:PP45_has_competency ?competence . - ?competence rdfs:label ?competenceLabel .} -UNION { <${subjectURL}> crmpe:PP4_hosts_object ?object .} -UNION { ?object crmpe:PP4i_is_object_hosted_by <${subjectURL}> .} - -UNION {<${subjectURL}> crmpe:PP31_uses_curation_plan ?curationplan .} -UNION { ?curationplan crmpe:PP31i_is_curation_plan_used_by <${subjectURL}> .} -UNION { - <${subjectURL}> crmpe:PP29_uses_access_protocol ?protocol . - ?protocol rdfs:label ?protocolLabel .} -UNION { - ?protocol crmpe:PP29i_is_access_protocol_used_by <${subjectURL}> . - ?protocol rdfs:label ?protocolLabel .} -UNION { <${subjectURL}> crmpe:PP32_curates ?object2 .} -UNION { ?object2 crmpe:PP32i_is_curated_by <${subjectURL}> .} -UNION { <${subjectURL}> crmpe:PP15_delivers_on_request ?delivered .} -UNION { ?delivered crmpe:PP15i_is_delivered_by <${subjectURL}>.} -UNION { <${subjectURL}> crmpe:PP14_runs_on_request ?ran .} -UNION { ?ran crmpe:PP14i_is_run_by <${subjectURL}>.} - UNION { - <${subjectURL}> crmpe:PP42_has_declarative_time ?declTime .} -UNION { - <${subjectURL}> crm:P16_used_specific_object ?Conditions_of_Use . - ?Conditions_of_Use a crm:E30_Right . - ?Conditions_of_Use crm:P3_has_note ?Conditions_of_Use_Note . -} -UNION { - <${subjectURL}> crm:P16_used_specific_object ?Conditions_of_Use . - ?Conditions_of_Use a crm:E30_Right . - ?Conditions_of_Use crm:P2_has_type ?Conditions_of_Use_Type . - ?Conditions_of_Use_Type rdfs:label ?Conditions_of_Use_Type_label . -} - -} - diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE35_Project.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE35_Project.sparql deleted file mode 100644 index 3695c2a..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/PE35_Project.sparql +++ /dev/null @@ -1,62 +0,0 @@ -DEFINE input:inference 'ariadneplus_rules' - -CONSTRUCT { - <${subjectURL}> rdfs:label ?label . - -<${subjectURL}> crm:P1_is_identified_by ?IDRes . -?IDRes a crm:E42_Identifier . -?IDRes rdfs:label ?ID_label . - -<${subjectURL}> crm:P102_has_title ?title . - - <${subjectURL}> crm:P3_has_note ?description . - - <${subjectURL}> a ?type . - - <${subjectURL}> crmpe:PP1_currently_offers ?offeredservice . - -<${subjectURL}> crmpe:PP44_has_maintaining_team ?team . - ?team crm:P76_has_contact_point ?contactpoint . - ?contactpoint crm:P2_has_type ?contactpointType . - ?contactpoint rdfs:label ?contactpointLabel . - ?contactpointType rdfs:label ?contactpointTypeLabel . - - <${subjectURL}> crm:P4_has_time-span ?time_span . - ?time_span crm:P82a_begin_of_the_begin ?time_actual . -} -WHERE { - <${subjectURL}> a crmpe:PE35_Project . - <${subjectURL}> a ?type . - { - <${subjectURL}> crm:P1_is_identified_by ?IDRes . - ?IDRes a crm:E42_Identifier . - ?IDRes rdfs:label ?ID_label . - } - UNION - { - <${subjectURL}> crm:P1_is_identified_by ?titleRes . - ?titleRes a crm:E41_Appellation . - ?titleRes rdfs:label ?title . - } -UNION{ <${subjectURL}> crm:P3_has_note ?description .} -UNION{ <${subjectURL}> rdfs:label ?label .} - -UNION{ - <${subjectURL}> crmpe:PP1_currently_offers ?offeredservice . -} -UNION{ - <${subjectURL}> crmpe:PP44_has_maintaining_team ?team . - } - UNION{ - <${subjectURL}> crmpe:PP44_has_maintaining_team ?team . - ?team crm:P76_has_contact_point ?contactpoint . - ?contactpoint crm:P2_has_type ?contactpointType . - ?contactpoint rdfs:label ?contactpointLabel . - ?contactpointType rdfs:label ?contactpointTypeLabel . - } -UNION { - <${subjectURL}> crm:P4_has_time-span ?time_span . - ?time_span crm:P82a_begin_of_the_begin ?time_actual . - } - -} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/rdfs_rule_set_isql.txt b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/rdfs_rule_set_isql.txt deleted file mode 100644 index e505fd8..0000000 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/parthenos/sparql/rdfs_rule_set_isql.txt +++ /dev/null @@ -1,76 +0,0 @@ -To check available rules: SELECT * from DB.DBA.SYS_RDF_SCHEMA; - -#### ARIADNEPLUS RULES #### - -### ORIGINAL WITH OFFICIALLY PUBLISHED SCHEMAS - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://www.cidoc-crm.org/cidoc-crm/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://ariadneplus.d4science.org/CRMext/CRMpe.rdfs/') ; - - -### USING SCHEMAS AVAILABLE FROM THE WORKSPACE - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://www.cidoc-crm.org/cidoc-crm/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('ariadneplus_rules', 'http://ariadneplus.d4science.org/CRMext/CRMpe.rdfs/') ; - -#### EXTENDED RULES #### - -# CRMext4SKOSandLabel_v1.4___23-03-2016180729___3031___20-09-2016163145___12893.rdfs -sparql load into ; -# CRMpc_v1.1.1.rdfs -sparql load into ; - -rdfs_rule_set ('extended_rules', 'http://www.cidoc-crm.org/cidoc-crm/') ; -rdfs_rule_set ('extended_rules', 'http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/') ; -rdfs_rule_set ('extended_rules', 'http://ariadneplus.d4science.org/CRMext/CRMpe.rdfs/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'https://vocabs.dariah.eu/bbt/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://www.cidoc-crm.org/cidoc-crm/CRMarchaeo/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://www.cidoc-crm.org/cidoc-crm/CRMgeo/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://www.cidoc-crm.org/cidoc-crm/CRMsci/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://iflastandards.info/ns/fr/frbr/frbroo/') ; - -SPARQL CLEAR GRAPH ; -sparql load into ; -rdfs_rule_set ('extended_rules', 'http://www.w3.org/2004/02/skos/core#') ; - - - - - - - - - diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java new file mode 100644 index 0000000..82b6154 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -0,0 +1,90 @@ +/** + * + */ +package eu.dnetlib.ariadneplus; + +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.ResourceManager; +import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import org.apache.commons.io.IOUtils; +import org.junit.Ignore; +import org.junit.Test; +import org.springframework.core.io.ClassPathResource; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +/** + * @author enrico.ottonello + * + */ +public class GraphDbReaderAndESIndexTest { + + private RunSPARQLQueryService runSPQRLQuery; + + @Test +// @Ignore + public void readAndIndexTest() throws Exception { + final ClassPathResource resource = new ClassPathResource("application.properties"); + Properties appProps = new Properties(); + appProps.load(resource.getInputStream()); + + runSPQRLQuery = new RunSPARQLQueryService(); + runSPQRLQuery.setupConnection( + appProps.getProperty("graphdb.writer.user"), + appProps.getProperty("graphdb.writer.pwd"), + appProps.getProperty("graphdb.serverUrl"), + appProps.getProperty("graphdb.repository")); + ParseRDFJSON parseRDFJSON = new ParseRDFJSON(); + parseRDFJSON.setCatalogEntryJsonPath(appProps.getProperty("catalog.entry.path")); + runSPQRLQuery.setParser(parseRDFJSON); + ResourceManager resourceManager = new ResourceManager(); + resourceManager.setup( + appProps.getProperty("type.path"), + appProps.getProperty("general.classpath"), + appProps.getProperty("exclude.predicates"), + appProps.getProperty("class.map.specifications") + ); + runSPQRLQuery.setResourceManager(resourceManager); + BulkUpload bulkUpload = new BulkUpload(); + bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname")); + runSPQRLQuery.setBulkUpload(bulkUpload); + String recordId = ""; + String datasource = "ads"; + String collectionId = "271"; + List recordIds = Arrays.asList(recordId); + final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); + String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); + runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId); + + } + + @Test + @Ignore + public void selectRecordsTest() throws Exception { + final ClassPathResource resource = new ClassPathResource("application.properties"); + Properties appProps = new Properties(); + appProps.load(resource.getInputStream()); + String datasource = "ads"; + String collectionId = "271"; + runSPQRLQuery = new RunSPARQLQueryService(); + runSPQRLQuery.setupConnection( + appProps.getProperty("graphdb.writer.user"), + appProps.getProperty("graphdb.writer.pwd"), + appProps.getProperty("repository.url"), + appProps.getProperty("graphdb.repository")); + runSPQRLQuery.selectRecordIds(datasource, collectionId); + } + + @Test + @Ignore + public void loadQueryTest() throws Exception { + final ClassPathResource resource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); + String queryTemplate = IOUtils.toString(resource.getInputStream(), StandardCharsets.UTF_8.name()); + System.out.println(queryTemplate); + + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java deleted file mode 100644 index ae7cd20..0000000 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * - */ -package eu.dnetlib.ariadneplus; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.springframework.beans.factory.annotation.Autowired; - -import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; -import eu.dnetlib.ariadneplus.reader.ResourceManager; -import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; -import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; - -/** - * @author enrico.ottonello - * - */ -@RunWith(JUnit4.class) -public class GraphDbReaderTest { - @Autowired - private RunSPARQLQueryService runSPQRLQuery; - @Autowired - private ParseRDFJSON parser; - @Autowired - private ResourceManager resourceManager; - @Autowired - private BulkUpload bulkUpload; - - @Test - public void readTest() throws Exception { - - runSPQRLQuery.setParser(parser); - runSPQRLQuery.setResourceManager(resourceManager); - runSPQRLQuery.setBulkUpload(bulkUpload); - runSPQRLQuery.executeQueryGraph(); - - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/IndexOnESJobNode.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/IndexOnESJobNode.java new file mode 100644 index 0000000..7704254 --- /dev/null +++ b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/IndexOnESJobNode.java @@ -0,0 +1,155 @@ +package eu.dnetlib.ariadneplus.workflows.nodes; + +import com.google.common.collect.Lists; +import eu.dnetlib.enabling.resultset.client.ResultSetClient; +import eu.dnetlib.msro.workflows.graph.Arc; +import eu.dnetlib.msro.workflows.nodes.AsyncJobNode; +import eu.dnetlib.msro.workflows.procs.Env; +import eu.dnetlib.msro.workflows.procs.Token; +import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider; +import eu.dnetlib.msro.workflows.util.WorkflowsConstants; +import eu.dnetlib.rmi.common.ResultSet; +import eu.dnetlib.rmi.manager.MSROException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.net.ConnectException; +import java.util.List; + + +public class IndexOnESJobNode extends AsyncJobNode { + + private static final Log log = LogFactory.getLog(IndexOnESJobNode.class); + + private String eprParam; + + @Autowired + private ResultSetClient resultSetClient; + + private String publisherEndpoint; + private String datasourceInterface; + private String datasource; + + //for parallel requests to the publisher endpoint + private int nThreads = 5; + + @Override + protected String execute(final Env env) throws Exception { + + int statusCode = -1; + String indexOnESResult = "noResult"; + log.info("Publisher endpoint: " + getPublisherEndpoint()); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(nThreads); + CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build(); + + log.info("IndexOnES endpoint: " + getIndexOnESEndpoint()); + CloseableHttpResponse responsePOST = null; + try { + HttpPost post = new HttpPost(getIndexOnESEndpoint()); + List params = Lists.newArrayList(); + String[] splits = getDatasourceInterface().split("::"); + String datasource = splits[2]; + String collectionId = splits[3]; + params.add(new BasicNameValuePair("datasource", datasource)); + params.add(new BasicNameValuePair("collectionId", collectionId)); + UrlEncodedFormEntity ent = new UrlEncodedFormEntity(params, "UTF-8"); + post.setEntity(ent); + log.info("Calling IndexOnES endpoint with params: "+getDatasource()+" "+getDatasourceInterface()); + responsePOST = client.execute(post); + statusCode = responsePOST.getStatusLine().getStatusCode(); + switch (statusCode) { + case 200: + log.info("index on ES completed"); + break; + default: + log.error("error indexing on ES " + responsePOST.getStatusLine().getStatusCode() + ": " + responsePOST.getStatusLine().getReasonPhrase()); + break; + } + } catch (ConnectException ce) { + log.error(ce); + throw new MSROException("Unable to connect to Publisher endpoint" + getIndexOnESEndpoint()); + } + catch (IOException e) { + log.error(e); + throw new MSROException("IO Error" + getIndexOnESEndpoint()); + } + finally{ + if(responsePOST != null) responsePOST.close(); + client.close(); + cm.shutdown(); + } + + env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "statusCode", Integer.toString(statusCode)); + env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "enrichResult", indexOnESResult); + + log.info(indexOnESResult); + if (statusCode!=200) { + throw new MSROException("Error from Publisher endpoint [ status code: " + statusCode + " ]"); + } + + return Arc.DEFAULT_ARC; + } + + public String getPublisherEndpoint() { + return publisherEndpoint; + } + + private String getIndexOnESEndpoint() { + return publisherEndpoint.concat("/indexOnES"); + } + + public void setPublisherEndpoint(final String publisherEndpoint) { + this.publisherEndpoint = publisherEndpoint; + } + + public ResultSetClient getResultSetClient() { + return resultSetClient; + } + + public void setResultSetClient(final ResultSetClient resultSetClient) { + this.resultSetClient = resultSetClient; + } + + public String getEprParam() { + return eprParam; + } + + public void setEprParam(String eprParam) { + this.eprParam = eprParam; + } + + public String getDatasourceInterface() { + return datasourceInterface; + } + + + public void setDatasourceInterface(String datasourceInterface) { + this.datasourceInterface = datasourceInterface; + } + + @Override + protected void beforeStart(Token token) { + token.setProgressProvider(new ResultsetProgressProvider(token.getEnv().getAttribute(getEprParam(), ResultSet.class), this.resultSetClient)); + } + + public String getDatasource() { + return datasource; + } + + public void setDatasource(String datasource) { + this.datasource = datasource; + } + +} diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml index cca0863..a3bb235 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml @@ -17,4 +17,6 @@ + + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st new file mode 100644 index 0000000..4ced6d1 --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st @@ -0,0 +1,54 @@ + + +
+ + + + + +
+ + $name$ + $desc$ + + aggregator + $priority$ + + + + + http://localhost:8080/ariadneplus/publish + + + + Index on Elastic Search all records of a collection from GraphDB + + + + + + + + + + + + + + + + + + + + + + + + + 9 9 9 ? * * + 10080 + + + + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml new file mode 100644 index 0000000..ef24ed1 --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml @@ -0,0 +1,32 @@ + +
+ + + + + +
+ + + + + + + http://localhost:8281/ariadneplus-graphdb + + + + Index on Elastic Search all records of a collection from GraphDB + + + + + + + + + + + + + \ No newline at end of file diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml new file mode 100644 index 0000000..668db14 --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml @@ -0,0 +1,60 @@ + + +
+ + + + + +
+ + Index On ES + Index on Elastic Search all records of a collection from GraphDB + + IndexOnES + Content Provider + + REPO_HI + 20 + + + + + Verify if DS is pending + + + + + + + + + + + Validate DS + + + + + + + Create Workflow + + + + + + + + + + + + + + 9 9 9 ? * * + 10080 + + + +