From 019bbd2f6a8bfd1549df2878064a5710b48a6eb9 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 28 Jan 2020 16:53:59 +0100 Subject: [PATCH] new classes to graphdb querying via sparql exewcution, data retrieving, feeding to elastic search --- dnet-ariadneplus-graphdb-publisher/pom.xml | 44 +- .../ariadneplus/elasticsearch/BulkUpload.java | 62 +++ .../elasticsearch/model/AatSubject.java | 39 ++ .../elasticsearch/model/AgentInfo.java | 75 +++ .../model/ArcheologicalResourceType.java | 52 ++ .../model/AriadneCatalogEntry.java | 453 ++++++++++++++++++ .../elasticsearch/model/AriadneGeoPoint.java | 49 ++ .../elasticsearch/model/AriadneTemporal.java | 75 +++ .../elasticsearch/model/DerivedSubject.java | 31 ++ .../ariadneplus/elasticsearch/model/Dex.java | 31 ++ .../elasticsearch/model/Distribution.java | 69 +++ .../model/ItemMetadataStructure.java | 22 + .../elasticsearch/model/MetadataRecord.java | 33 ++ .../elasticsearch/model/NativeSubject.java | 53 ++ .../elasticsearch/model/Spatial.java | 136 ++++++ .../ariadneplus/reader/ResourceManager.java | 127 +++++ .../reader/RunSPARQLQueryService.java | 142 ++++++ .../ariadneplus/reader/json/ParseRDFJSON.java | 66 +++ .../ariadneplus/reader/utils/ClassSpec.java | 44 ++ .../ariadneplus/reader/utils/Mappings.java | 58 +++ .../reader/utils/PropertiesMap.java | 33 ++ .../src/main/resources/application.properties | 338 ++++++++++++- .../ariadneplus/GraphDbReaderTest.java | 40 ++ 23 files changed, 2062 insertions(+), 10 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AatSubject.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DerivedSubject.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Dex.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Distribution.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ItemMetadataStructure.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/MetadataRecord.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ClassSpec.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/Mappings.java create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java create mode 100644 dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java diff --git a/dnet-ariadneplus-graphdb-publisher/pom.xml b/dnet-ariadneplus-graphdb-publisher/pom.xml index 8ca3baf..62f32e4 100644 --- a/dnet-ariadneplus-graphdb-publisher/pom.xml +++ b/dnet-ariadneplus-graphdb-publisher/pom.xml @@ -1,11 +1,10 @@ - org.springframework.boot - spring-boot-starter-parent - 1.5.6.RELEASE - - + org.springframework.boot + spring-boot-starter-parent + 2.1.3.RELEASE + 4.0.0 eu.dnetlib @@ -60,10 +59,11 @@ - junit - junit - ${junit.version} - + junit + junit + 4.13-rc-1 + test + net.sf.saxon Saxon-HE @@ -101,6 +101,32 @@ graphdb-free-runtime 9.0.0 + + com.google.code.gson + gson + 2.8.6 + + + com.jayway.jsonpath + json-path + 2.4.0 + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 7.4.2 + + + org.elasticsearch + elasticsearch + 7.4.2 + + + + org.elasticsearch.client + elasticsearch-rest-client + 7.4.2 + diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java new file mode 100644 index 0000000..2d5c6d9 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -0,0 +1,62 @@ +package eu.dnetlib.ariadneplus.elasticsearch; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; + +import javax.annotation.PostConstruct; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry; +import eu.dnetlib.ariadneplus.reader.ResourceManager; + +@Service +public class BulkUpload { + + @Value("${elasticsearch.url:localhost:9200}") + private String elasticsearchUrl; + + private RestHighLevelClient client; + + @PostConstruct + private void init() throws IOException { + client = new RestHighLevelClient( + RestClient.builder( + new HttpHost("localhost",9200,"http"))); + + } + + public void index(ResourceManager manager) { + BulkRequest request = new BulkRequest(); + while (manager.hasNext()){ + try { + AriadneCatalogEntry ace = ((AriadneCatalogEntry) manager.next()); + request.add(new IndexRequest("prova_via_code").id(ace.getOriginalId()) + .source(ace.toJson(),XContentType.JSON)); + System.out.println("indexing to ES record "+ace.getOriginalId()); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (InstantiationException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AatSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AatSubject.java new file mode 100644 index 0000000..1157902 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AatSubject.java @@ -0,0 +1,39 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; +import com.google.gson.*; + +public class AatSubject { + private String id; + private String label; + private String lang; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getLang() { + return lang; + } + + public void setLang(String lang) { + this.lang = lang; + } + + public AatSubject() { + } + + public static AatSubject fromJson(String json){ + return new Gson().fromJson(json, AatSubject.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java new file mode 100644 index 0000000..9a885a1 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java @@ -0,0 +1,75 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + +public class AgentInfo { + private String email; + private String name; + private String phone; + private String type; + + public AgentInfo() { + + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPhone() { + return phone; + } + + public void setPhone(String phone) { + this.phone = phone; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public static AgentInfo fromJson(String json){ + return new Gson().fromJson(json, AgentInfo.class); + } + + public static AgentInfo fromRDFJson(JsonElement json){ + AgentInfo pi = new AgentInfo(); + for (Map.Entry entry : json.getAsJsonObject().entrySet()){ + switch (entry.getKey()){ + case "http://www.myprefix/name" : + pi.setName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/type": + pi.setType(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/email": + pi.setEmail(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/phone": + pi.setPhone(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + } + } + + return pi; + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java new file mode 100644 index 0000000..80ea18c --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java @@ -0,0 +1,52 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + +public class ArcheologicalResourceType { + private String id; + private String name; + + public ArcheologicalResourceType() { + } + + public static ArcheologicalResourceType fromRDFJson(JsonElement json) { + ArcheologicalResourceType art = new ArcheologicalResourceType(); + for (Map.Entry entry : json.getAsJsonObject().entrySet()){ + switch (entry.getKey()){ + case "http://www.myprefix/id" : + art.setId(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/name": + String tmp = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); + art.setName(tmp); + break; + + } + } + + return art; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public static ArcheologicalResourceType fromJson(String json){ + return new Gson().fromJson(json, ArcheologicalResourceType.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java new file mode 100644 index 0000000..938868f --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java @@ -0,0 +1,453 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import java.util.List; + + +public class AriadneCatalogEntry { + private List aatSubjects; + private String accessPolicy; + private String accessRights; + private ArcheologicalResourceType archeologicalResourceType; + private String contactPoint; + private List contributor; + private List creator; + private List derivedSubject; + private String description; + private List distribution; + private List publisher; + private String title; + private String extent; + private List hasItemMetadataStructure; + private List hasMetadataRecord; + private String identifier; + private String partOf; + private String issued; + private List keyword; + private String landingPage; + private String language; + private List legalResponsible; + private String modified; + private List nativeSubject; + private String originalId; + private List owner; + private String packageId; + private String placeName; + private String postcode; + private String providerId; + private String rdfAbout; + private String resourceType; + private String rights; + private List scientificResponsible; + private List spatial; + private List technicalResponsible; + private List temporal; + + public List getContributor() { + return contributor; + } + + public void setContributor(List contributor) { + this.contributor = contributor; + } + + + + public List getAatSubjects() { + return aatSubjects; + } + + public void setAatSubjects(List aatSubjects) { + this.aatSubjects = aatSubjects; + } + + public String getAccessPolicy() { + return accessPolicy; + } + + public void setAccessPolicy(String accessPolicy) { + this.accessPolicy = accessPolicy; + } + + public String getAccessRights() { + return accessRights; + } + + public void setAccessRights(String accessRights) { + this.accessRights = accessRights; + } + + public ArcheologicalResourceType getArcheologicalResourceType() { + return archeologicalResourceType; + } + + public void setArcheologicalResourceType(ArcheologicalResourceType archeologicalResourceType) { + this.archeologicalResourceType = archeologicalResourceType; + } + + public String getContactPoint() { + return contactPoint; + } + + public void setContactPoint(String contactPoint) { + this.contactPoint = contactPoint; + } + + public List getCreator() { + return creator; + } + + public void setCreator(List creator) { + this.creator = creator; + } + + public List getDerivedSubject() { + return derivedSubject; + } + + public void setDerivedSubject(List derivedSubject) { + this.derivedSubject = derivedSubject; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getDistribution() { + return distribution; + } + + public void setDistribution(List distribution) { + this.distribution = distribution; + } + + public List getPublisher() { + return publisher; + } + + public void setPublisher(List publisher) { + this.publisher = publisher; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getExtent() { + return extent; + } + + public void setExtent(String extent) { + this.extent = extent; + } + + public List getHasItemMetadataStructure() { + return hasItemMetadataStructure; + } + + public void setHasItemMetadataStructure(List hasItemMetadataStructure) { + this.hasItemMetadataStructure = hasItemMetadataStructure; + } + + public List getHasMetadataRecord() { + return hasMetadataRecord; + } + + public void setHasMetadataRecord(List hasMetadataRecord) { + this.hasMetadataRecord = hasMetadataRecord; + } + + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public String getPartOf() { + return partOf; + } + + public void setPartOf(String partOf) { + this.partOf = partOf; + } + + public String getIssued() { + return issued; + } + + public void setIssued(String issued) { + this.issued = issued; + } + + public List getKeyword() { + return keyword; + } + + public void setKeyword(List keyword) { + this.keyword = keyword; + } + + public String getLandingPage() { + return landingPage; + } + + public void setLandingPage(String landingPage) { + this.landingPage = landingPage; + } + + public String getLanguage() { + return language; + } + + public void setLanguage(String language) { + this.language = language; + } + + public List getLegalResponsible() { + return legalResponsible; + } + + public void setLegalResponsible(List legalResponsible) { + this.legalResponsible = legalResponsible; + } + + public String getModified() { + return modified; + } + + public void setModified(String modified) { + this.modified = modified; + } + + public List getNativeSubject() { + return nativeSubject; + } + + public void setNativeSubject(List nativeSubject) { + this.nativeSubject = nativeSubject; + } + + public String getOriginalId() { + return originalId; + } + + public void setOriginalId(String originalId) { + this.originalId = originalId; + } + + public List getOwner() { + return owner; + } + + public void setOwner(List owner) { + this.owner = owner; + } + + public String getPackageId() { + return packageId; + } + + public void setPackageId(String packageId) { + this.packageId = packageId; + } + + public String getPlaceName() { + return placeName; + } + + public void setPlaceName(String placeName) { + this.placeName = placeName; + } + + public String getPostcode() { + return postcode; + } + + public void setPostcode(String postcode) { + this.postcode = postcode; + } + + public String getProviderId() { + return providerId; + } + + public void setProviderId(String providerId) { + this.providerId = providerId; + } + + public String getRdfAbout() { + return rdfAbout; + } + + public void setRdfAbout(String rdfAbout) { + this.rdfAbout = rdfAbout; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public String getRights() { + return rights; + } + + public void setRights(String rights) { + this.rights = rights; + } + + public List getScientificResponsible() { + return scientificResponsible; + } + + public void setScientificResponsible(List scientificResponsible) { + this.scientificResponsible = scientificResponsible; + } + + public List getSpatial() { + return spatial; + } + + public void setSpatial(List spatial) { + this.spatial = spatial; + } + + public List getTechnicalResponsible() { + return technicalResponsible; + } + + public void setTechnicalResponsible(List technicalResponsible) { + this.technicalResponsible = technicalResponsible; + } + + public List getTemporal() { + return temporal; + } + + public void setTemporal(List temporal) { + this.temporal = temporal; + } + + + + public static AriadneCatalogEntry fromJson(String json){ + return new Gson().fromJson(json, AriadneCatalogEntry.class); + } + + public String toJson(){ + return new Gson().toJson(this); + } + +// public static AriadneCatalogEntry fromRDFJson(JsonElement json, String identifier, Map map){ +// AriadneCatalogEntry acim = new AriadneCatalogEntry(); +// acim.setIdentifier(identifier.substring(identifier.lastIndexOf("/") + 1)); +// JsonObject content = json.getAsJsonObject(); +// for (Map.Entry stringJsonElementEntry : content.entrySet()) { +// switch (stringJsonElementEntry.getKey()){ +// case "http://www.myprefix/accessPolicy": +// acim.setAccessPolicy(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/accessRights": +// acim.setAccessRights(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/contributor": +// JsonArray contributor_array = stringJsonElementEntry.getValue().getAsJsonArray(); +// List contributor_list = new ArrayList(); +// for (int i = 0; i < contributor_array.size() ; i++ ){ +// String map_key = contributor_array.get(i).getAsJsonObject().get("value").getAsString(); +// contributor_list.add(AgentInfo.fromRDFJson(map.get(map_key))); +// +// } +// acim.setContributor(contributor_list); +// break; +// case "http://www.myprefix/description": +// acim.setDescription(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/isPartOf": +// acim.setPartOf(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/issued": +// acim.setIssued(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/landingPage": +// acim.setLandingPage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/language": +// acim.setLanguage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/modified": +// acim.setModified(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/nativeSubject": +// JsonArray nativeSubject_array = stringJsonElementEntry.getValue().getAsJsonArray(); +// List nativeSubject_list = new ArrayList(); +// for (int i = 0; i < nativeSubject_array.size() ; i++ ){ +// String map_key = nativeSubject_array.get(i).getAsJsonObject().get("value").getAsString(); +// nativeSubject_list.add(NativeSubject.fromRDFJson(map.get(map_key))); +// +// } +// acim.setNativeSubject(nativeSubject_list); +// break; +// case "http://www.myprefix/originalId": +// acim.setOriginalId(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/resourceType": +// acim.setResourceType(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/spatial": +// JsonArray spatial_array = stringJsonElementEntry.getValue().getAsJsonArray(); +// List spatial_list = new ArrayList(); +// for (int i = 0; i < spatial_array.size() ; i++ ){ +// String map_key = spatial_array.get(i).getAsJsonObject().get("value").getAsString(); +// spatial_list.add(Spatial.fromRDFJson(map.get(map_key), map)); +// +// } +// acim.setSpatial(spatial_list); +// break; +// case "http://www.myprefix/temporal": +// JsonArray temporal_array = stringJsonElementEntry.getValue().getAsJsonArray(); +// List temporal_list = new ArrayList<>(); +// for(int i=0; i < temporal_array.size(); i++){ +// String map_key = temporal_array.get(i).getAsJsonObject().get("value").getAsString(); +// temporal_list.add(AriadneTemporal.fromRDFJson(map.get(map_key))); +// } +// acim.setTemporal(temporal_list); +// break; +// case "http://www.myprefix/title": +// acim.setTitle(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); +// break; +// case "http://www.myprefix/publisher": +// JsonArray publisher_array = stringJsonElementEntry.getValue().getAsJsonArray(); +// List publisher_list = new ArrayList(); +// for (int i = 0; i < publisher_array.size() ; i++ ){ +// String map_key = publisher_array.get(i).getAsJsonObject().get("value").getAsString(); +// publisher_list.add(AgentInfo.fromRDFJson(map.get(map_key))); +// +// } +// acim.setPublisher(publisher_list); +// break; +// case "http://www.myprefix/archeologicalResourceType": +// acim.setArcheologicalResourceType(ArcheologicalResourceType.fromRDFJson(map.get(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()))); +// } +// +// +// } +// +// System.out.println(acim.toJson()); +// return acim; +// } +} +//https://ariadne-infrastructure.eu/aocat \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java new file mode 100644 index 0000000..2b6b295 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java @@ -0,0 +1,49 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + +public class AriadneGeoPoint { + private String lat; + private String lon; + + public String getLat() { + return lat; + } + + public void setLat(String lat) { + this.lat = lat; + } + + public String getLon() { + return lon; + } + + public void setLon(String lon) { + this.lon = lon; + } + + public AriadneGeoPoint() { + } + + public static AriadneGeoPoint fromJson (String json){ + return new Gson().fromJson(json, AriadneGeoPoint.class); + } + + public static AriadneGeoPoint fromRDFJson(JsonElement json){ + AriadneGeoPoint agp = new AriadneGeoPoint(); + for (Map.Entry stringJsonElementEntry : json.getAsJsonObject().entrySet()) { + switch (stringJsonElementEntry.getKey()){ + case "http://www.myprefix/lat": + agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/lon": + agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + } + } + return agp; + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java new file mode 100644 index 0000000..115b4fa --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java @@ -0,0 +1,75 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + +public class AriadneTemporal { + private String from; + private String periodName; + private String until; + private String uri; + + public static AriadneTemporal fromRDFJson(JsonElement json) { + AriadneTemporal at = new AriadneTemporal(); + for (Map.Entry entry : json.getAsJsonObject().entrySet()){ + switch (entry.getKey()){ + case "http://www.myprefix/from" : + at.setFrom(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/periodName": + at.setPeriodName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/until": + at.setUntil(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/uri": + at.setUri(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + + } + } + + return at; + } + + public String getFrom() { + return from; + } + + public void setFrom(String from) { + this.from = from; + } + + public String getPeriodName() { + return periodName; + } + + public void setPeriodName(String periodName) { + this.periodName = periodName; + } + + public String getUntil() { + return until; + } + + public void setUntil(String until) { + this.until = until; + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public AriadneTemporal() { + } + + public static AriadneTemporal fromJson(String json){ + return new Gson().fromJson(json, AriadneTemporal.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DerivedSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DerivedSubject.java new file mode 100644 index 0000000..2ea7221 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/DerivedSubject.java @@ -0,0 +1,31 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class DerivedSubject { + private String prefLabel; + private String source; + + public String getPrefLabel() { + return prefLabel; + } + + public void setPrefLabel(String prefLabel) { + this.prefLabel = prefLabel; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public DerivedSubject() { + } + + public static DerivedSubject fromJson(String json){ + return new Gson().fromJson(json, DerivedSubject.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Dex.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Dex.java new file mode 100644 index 0000000..5784e03 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Dex.java @@ -0,0 +1,31 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class Dex { + private String characterSet; + private String description; + + public String getCharacterSet() { + return characterSet; + } + + public void setCharacterSet(String characterSet) { + this.characterSet = characterSet; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Dex() { + } + + public static Dex fromJson(String json){ + return new Gson().fromJson(json,Dex.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Distribution.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Distribution.java new file mode 100644 index 0000000..cd9d163 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Distribution.java @@ -0,0 +1,69 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +import java.util.List; + +public class Distribution { + private String accessURL; + private String description; + private String issued; + private String modified; + private List publisher; + private String title; + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public List getPublisher() { + return publisher; + } + + public void setPublisher(List publisher) { + this.publisher = publisher; + } + + public String getAccessURL() { + return accessURL; + } + + public void setAccessURL(String accessURL) { + this.accessURL = accessURL; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getIssued() { + return issued; + } + + public void setIssued(String issued) { + this.issued = issued; + } + + public String getModified() { + return modified; + } + + public void setModified(String modified) { + this.modified = modified; + } + + public Distribution() { + } + + public static Distribution fromJson(String json){ + return new Gson().fromJson(json,Distribution.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ItemMetadataStructure.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ItemMetadataStructure.java new file mode 100644 index 0000000..b344686 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ItemMetadataStructure.java @@ -0,0 +1,22 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +public class ItemMetadataStructure { + private String characterSet; + + public String getCharacterSet() { + return characterSet; + } + + public void setCharacterSet(String characterSet) { + this.characterSet = characterSet; + } + + public ItemMetadataStructure() { + } + + private static ItemMetadataStructure fromJson(String json){ + return new Gson().fromJson(json, ItemMetadataStructure.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/MetadataRecord.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/MetadataRecord.java new file mode 100644 index 0000000..8a02e9e --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/MetadataRecord.java @@ -0,0 +1,33 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; + +import java.util.List; + +public class MetadataRecord { + private List conformsTo; + private String xmlDoc; + + public List getConformsTo() { + return conformsTo; + } + + public void setConformsTo(List conformsTo) { + this.conformsTo = conformsTo; + } + + public String getXmlDoc() { + return xmlDoc; + } + + public void setXmlDoc(String xmlDoc) { + this.xmlDoc = xmlDoc; + } + + public MetadataRecord() { + } + + public static MetadataRecord fromJson(String json){ + return new Gson().fromJson(json, MetadataRecord.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java new file mode 100644 index 0000000..324eca8 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java @@ -0,0 +1,53 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + +public class NativeSubject { + private String prefLabel; + private String rdfAbout; + + + + public String getPrefLabel() { + return prefLabel; + } + + public void setPrefLabel(String prefLabel) { + this.prefLabel = prefLabel; + } + + public String getRdfAbout() { + return rdfAbout; + } + + public void setRdfAbout(String rdfAbout) { + this.rdfAbout = rdfAbout; + } + + public NativeSubject() { + } + + public static NativeSubject fromJson(String json){ + return new Gson().fromJson(json, NativeSubject.class); + } + + public static NativeSubject fromRDFJson(JsonElement json) { + NativeSubject pi = new NativeSubject(); + for (Map.Entry entry : json.getAsJsonObject().entrySet()){ + switch (entry.getKey()){ + case "http://www.myprefix/prefLabel" : + pi.setPrefLabel(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/rdfAbout": + pi.setRdfAbout(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + + } + } + + return pi; + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java new file mode 100644 index 0000000..0587975 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java @@ -0,0 +1,136 @@ +package eu.dnetlib.ariadneplus.elasticsearch.model; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.util.Map; + + +public class Spatial { + private String address; + private String boundingBoxMaxLat; + private String boundingBoxMaxLon; + private String boundingBoxMinLat; + private String boundingBoxMinLon; + private String coordinateSystem; + private String country; + private AriadneGeoPoint location; + private String placeName; + + public static Spatial fromRDFJson(JsonElement json, Map map) { + Spatial pi = new Spatial(); + for (Map.Entry entry : json.getAsJsonObject().entrySet()){ + switch (entry.getKey()){ + case "http://www.myprefix/address" : + pi.setAddress(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/boundingBoxMaxLat": + pi.setBoundingBoxMaxLat(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/boundingBoxMaxLon": + pi.setBoundingBoxMaxLon(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/boundingBoxMinLat": + pi.setBoundingBoxMinLat(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/boundingBoxMinLon": + pi.setBoundingBoxMinLon(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/country": + pi.setCountry(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/coordinateSystem": + pi.setCoordinateSystem(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + case "http://www.myprefix/location": + String map_key = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString(); + pi.setLocation(AriadneGeoPoint.fromRDFJson(map.get(map_key))); + break; + case "http://www.myprefix/placeName": + pi.setPlaceName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString()); + break; + } + } + + return pi; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getBoundingBoxMaxLat() { + return boundingBoxMaxLat; + } + + public void setBoundingBoxMaxLat(String boundingBoxMaxLat) { + this.boundingBoxMaxLat = boundingBoxMaxLat; + } + + public String getBoundingBoxMaxLon() { + return boundingBoxMaxLon; + } + + public void setBoundingBoxMaxLon(String boundingBoxMaxLon) { + this.boundingBoxMaxLon = boundingBoxMaxLon; + } + + public String getBoundingBoxMinLat() { + return boundingBoxMinLat; + } + + public void setBoundingBoxMinLat(String boundingBoxMinLat) { + this.boundingBoxMinLat = boundingBoxMinLat; + } + + public String getBoundingBoxMinLon() { + return boundingBoxMinLon; + } + + public void setBoundingBoxMinLon(String boundingBoxMinLon) { + this.boundingBoxMinLon = boundingBoxMinLon; + } + + public String getCoordinateSystem() { + return coordinateSystem; + } + + public void setCoordinateSystem(String coordinateSystem) { + this.coordinateSystem = coordinateSystem; + } + + public String getCountry() { + return country; + } + + public void setCountry(String country) { + this.country = country; + } + + public AriadneGeoPoint getLocation() { + return location; + } + + public void setLocation(AriadneGeoPoint location) { + this.location = location; + } + + public String getPlaceName() { + return placeName; + } + + public void setPlaceName(String placeName) { + this.placeName = placeName; + } + + public Spatial() { + } + + public static Spatial fromJson(String json){ + return new Gson().fromJson(json, Spatial.class); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java new file mode 100644 index 0000000..732fe78 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java @@ -0,0 +1,127 @@ +package eu.dnetlib.ariadneplus.reader; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Set; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; + +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; +import eu.dnetlib.ariadneplus.reader.utils.ClassSpec; +import eu.dnetlib.ariadneplus.reader.utils.Mappings; +import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap; +import net.minidev.json.JSONArray; +import net.minidev.json.JSONObject; + +@Service +public class ResourceManager { + + @Value("${type.path:undefined}") + private String type_path; + @Value("${general.classpath:undefined}") + private String general_classpath; + @Value("${exclude.predicates:[]}") + private String exclude_predicates; + + private List not_parsable; + + private ParseRDFJSON parser; + + @PostConstruct + public void init(){ + Type listType = new TypeToken>(){}.getType(); + not_parsable = new Gson().fromJson(exclude_predicates, listType); + } + + + private String getFieldValue(Object value){ + + if (value instanceof LinkedHashMap) + return (String)((LinkedHashMap)value).get("value"); + return (String)((JSONObject)value).get("value"); + + } + + public void manage(ParseRDFJSON parser){ + this.parser = parser; + } + + public boolean hasNext(){ + return parser.hasNextElement(); + } + + public Object next() throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException { + return manage(parser.getNextElement(),null); + } + + private Object manage(Object entry, String class_name) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, InstantiationException { + if (class_name == null){ + if(entry instanceof LinkedHashMap){ + LinkedHashMap tmp = (LinkedHashMap)((JSONArray)((LinkedHashMap)entry).get(type_path)).get(0); + class_name = (String)tmp.get("value"); + } + } + Class c = Class.forName(general_classpath + class_name); + Object class_instance = c.newInstance(); + ClassSpec class_spec = PropertiesMap.get(class_name); + Set keySet; + + if(entry instanceof LinkedHashMap) + keySet = ((LinkedHashMap)entry).keySet(); + else + keySet = ((JSONObject)entry).keySet(); + + for(Object predicate: keySet){//predicates in the json + if (not_parsable.contains(predicate)) + continue; + Mappings map = class_spec.get((String)predicate); + + if (map==null) { + continue; + } + JSONArray values; + + if(entry instanceof LinkedHashMap) + values = (JSONArray)((LinkedHashMap)entry).get(predicate); + else + values = (JSONArray)((JSONObject)entry).get(predicate); + + if (!map.hasExternalReference()){ + Method setField = c.getMethod("set" + map.getClass_field(), Class.forName(map.getElement_type())); + setField.invoke(class_instance, getFieldValue(values.get(0))); + } + else{ + if(PropertiesMap.get(map.getExternal_reference()).getClass_type().equals("prototype")){ + List value_list = new ArrayList<>(); + for(Object value: values){ + value_list.add(manage(ParseRDFJSON.get(getFieldValue(value)), map.getExternal_reference())); + } + Method setField = c.getMethod("set" + map.getClass_field(),List.class); + setField.invoke(class_instance, value_list); + } + else{ + Class ref = Class.forName(general_classpath + map.getExternal_reference()); + Method setField = c.getMethod("set" + map.getClass_field(), ref); + setField.invoke(class_instance, manage (ParseRDFJSON.get(getFieldValue(values.get(0))),map.getExternal_reference())); + } + + } + + } + + return class_instance; + } + + + +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java new file mode 100644 index 0000000..749ec67 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -0,0 +1,142 @@ +package eu.dnetlib.ariadneplus.reader; + +import java.io.StringWriter; + +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Literal; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.ValueFactory; +import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.query.GraphQuery; +import org.eclipse.rdf4j.query.GraphQueryResult; +import org.eclipse.rdf4j.query.QueryLanguage; +import org.eclipse.rdf4j.query.QueryResults; +import org.eclipse.rdf4j.repository.Repository; +import org.eclipse.rdf4j.repository.RepositoryConnection; +import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.eclipse.rdf4j.rio.RDFWriter; +import org.eclipse.rdf4j.rio.Rio; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; + +@Service +public class RunSPARQLQueryService { + + @Value("${sparql.query3:undefined}") + private String query3; + + @Value("${repository.url:undefined}") + private String repository_url; + + private RepositoryConnection connection; + private RemoteRepositoryManager manager; + private Repository repository; + + private ParseRDFJSON parser; + private ResourceManager resourceManager; + private BulkUpload bulkUpload; + + private void openConnection(){ + manager = new RemoteRepositoryManager(repository_url); + manager.init(); + manager.setUsernameAndPassword("writer", "Writer01"); +// repository = manager.getRepository("test01"); + repository = manager.getRepository("ariadneplus-ts01"); + connection = repository.getConnection(); + + } + + private void closeConnection(){ + connection.close(); + repository.shutDown(); + manager.shutDown(); + } + + public String executeQueryGraph(){ + if (query3.equals("undefined")) + return null; + openConnection(); + StringWriter recordWriter = null; + Model model = null; + String jsonRecord = null; + try { + GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query3); + GraphQueryResult graphQueryResult = graphQuery.evaluate(); + System.out.println("as model ..."); + model = QueryResults.asModel(graphQueryResult); + graphQueryResult.close(); + int resourceCount = 0; + System.out.println("counting resources ... " ); + ValueFactory factory = SimpleValueFactory.getInstance(); + IRI iri = factory.createIRI("http://www.myprefix/resourcetype"); + Literal value = factory.createLiteral("AriadneCatalogEntry"); + for (Resource record: model.filter(null, iri, value).subjects()) { + System.out.println(); + Model recordModel = null; +// RDFWriter rdfRecordWriter = null; + resourceCount+=1; +// if (resourceCount==12) { +// break; +// } + recordModel = model.filter(record, null, null); + if (recordModel!=null && !recordModel.isEmpty()) { + recordWriter = new StringWriter(); + RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); + Rio.write(recordModel, rdfRecordWriter); +// System.out.println("record json: "+ recordWriter.toString()); + parser.parse(recordWriter.toString()); + resourceManager.manage(parser); + bulkUpload.index(resourceManager); + } + } + System.out.println(); + System.out.println("count resources: "+ resourceCount); + System.out.println("count statements: " + model.size()); + System.out.println("index to Elastic Search completed"); + }catch(Exception e){ + e.printStackTrace(); + }finally{ + closeConnection(); + if (model!=null) { + model.clear(); + } + } + return jsonRecord; + } + + + public ParseRDFJSON getParser() { + return parser; + } + + + public void setParser(ParseRDFJSON parser) { + this.parser = parser; + } + + + public ResourceManager getResourceManager() { + return resourceManager; + } + + + public void setResourceManager(ResourceManager resourceManager) { + this.resourceManager = resourceManager; + } + + + public BulkUpload getBulkUpload() { + return bulkUpload; + } + + + public void setBulkUpload(BulkUpload bulkUpload) { + this.bulkUpload = bulkUpload; + } + +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java new file mode 100644 index 0000000..89ca81a --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java @@ -0,0 +1,66 @@ +package eu.dnetlib.ariadneplus.reader.json; + +import java.util.Iterator; +import java.util.LinkedHashMap; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; + +import net.minidev.json.JSONArray; +import net.minidev.json.JSONObject; +import net.minidev.json.parser.JSONParser; +import net.minidev.json.parser.ParseException; + + +@Service +public class ParseRDFJSON { + + static JSONObject map ; + + @Value("${catalog.entry.path:undefined}") + private String query; + + private String json; + + private Iterator it ; + + public String getJson() { + return json; + } + + + public void setJson(String json) throws ParseException { + this.json = json; + } + + private void fillMap() throws ParseException { + map = (JSONObject)(new JSONParser(JSONParser.MODE_PERMISSIVE).parse(json)); + + } + + public void parse(String json) throws ParseException { + setJson(json); + fillMap(); + DocumentContext jsonContext = JsonPath.parse(json); + JSONArray entries = jsonContext.read(query); + int size = entries.size(); + it = entries.iterator(); + } + + public boolean hasNextElement(){ + return it.hasNext(); + } + + public LinkedHashMap getNextElement(){ + return (LinkedHashMap)it.next(); + } + + public static JSONObject get(String key){ + return (JSONObject) map.get(key); + } + + +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ClassSpec.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ClassSpec.java new file mode 100644 index 0000000..260f40a --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ClassSpec.java @@ -0,0 +1,44 @@ +package eu.dnetlib.ariadneplus.reader.utils; + +import com.google.gson.Gson; + +import java.util.List; +import java.util.Map; + +public class ClassSpec { + private String class_type; + private Map mappings; + + public ClassSpec() { + } + + + public String getClass_type() { + return class_type; + } + + public void setClass_type(String class_type) { + this.class_type = class_type; + } + + public Map getMappings() { + return mappings; + } + + public void setMappings(Map mappings) { + this.mappings = mappings; + } + + public static ClassSpec fromJson(String json){ + return new Gson().fromJson(json, ClassSpec.class); + } + + public String toJson(){ + return new Gson().toJson(this); + } + + + public Mappings get(String predicate) { + return mappings.get(predicate); + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/Mappings.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/Mappings.java new file mode 100644 index 0000000..bab8418 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/Mappings.java @@ -0,0 +1,58 @@ +package eu.dnetlib.ariadneplus.reader.utils; + +import com.google.gson.Gson; + +public class Mappings { + private String class_field; + private String substring; + private String element_type; + private String external_reference; + + public Mappings() { + } + + + public String getClass_field() { + return class_field; + } + + public void setClass_field(String class_field) { + this.class_field = class_field; + } + + public String getSubstring() { + return substring; + } + + public void setSubstring(String substring) { + this.substring = substring; + } + + public String getElement_type() { + return element_type; + } + + public void setElement_type(String element_type) { + this.element_type = element_type; + } + + public String getExternal_reference() { + return external_reference; + } + + public void setExternal_reference(String external_reference) { + this.external_reference = external_reference; + } + + public static Mappings fromJson(String json){ + return new Gson().fromJson(json, Mappings.class); + } + + public String toJson(){ + return new Gson().toJson(this); + } + + public boolean hasExternalReference() { + return external_reference != null ; + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java new file mode 100644 index 0000000..897f3e4 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java @@ -0,0 +1,33 @@ +package eu.dnetlib.ariadneplus.reader.utils; + +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; + +@Component +public class PropertiesMap { + + @Value("${class.map.specifications:undefined}") + private String spec; + + private static Map map; + + public static ClassSpec get(String key){ + return map.get(key); + } + + @PostConstruct + public void fill(){ + Type mapType = new TypeToken>(){}.getType(); + map = new Gson().fromJson(spec,mapType); + + } +} diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties index a5b6d00..43e9c37 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties @@ -7,4 +7,340 @@ graphdb.writer.pwd=writer01 graphdb.repository=ariadneprova graphdb.sparqlUrl = http://localhost:7200/sparql -graphdb.baseURI=https://ariadne-infrastructure.eu/ \ No newline at end of file +graphdb.baseURI=https://ariadne-infrastructure.eu/ + +#repository.url=http://localhost:7200 +repository.url=http://graphdb-test.ariadne.d4science.org:7200 + +sparql.query2=PREFIX skos: PREFIX rdfs: PREFIX aocat: PREFIX mine: CONSTRUCT { ?archeologicalResourceType a mine:archeologicalResourceType . ?record mine:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType mine:name ?archeologicalResourceTypeName . } WHERE { GRAPH { ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . } } + +sparql.query=PREFIX aocat: \ +PREFIX rdfs: \ +PREFIX skos: \ +PREFIX mine: \ +CONSTRUCT { \ + ?record mine:resourcetype "AriadneCatalogEntry" . \ + ?record mine:identifier ?record .\ + ?record mine:publisher ?x . \ + ?x a mine:AgentInfo . \ + ?x mine:name ?publisherName . \ + ?x mine:type ?publisherType . \ + ?archeologicalResourceType a mine:archeologicalResourceType . \ +?record mine:archeologicalResourceType ?archeologicalResourceType . \ +?archeologicalResourceType mine:name ?archeologicalResourceTypeName . \ +?record mine:issued ?issued . \ +?record mine:resourceType ?resourceType . \ +?record mine:modified ?modified . \ +?record mine:nativeSubject ?ns . \ + ?ns a mine:NativeSubject . \ +?ns mine:prefLabel ?nativeSbj . \ + ?spatial a mine:Spatial . \ +?record mine:spatial ?spatial . \ +?spatial mine:placeName ?spatialPlaceName . \ +?spatial mine:location ?blocation . \ + ?blocation a mine:AriadneGeoPoint . \ +?blocation mine:lat ?spatialLocationLat . \ +?blocation mine:lon ?spatialLocationLon . \ +?record mine:accessPolicy ?accessPolicy . \ +?record mine:landingPage ?landingPage . \ +?record mine:title ?title . \ +?record mine:accessRights ?accessRights . \ +?record mine:description ?description . \ +?record mine:contributor ?contributor . \ + ?contributor a mine:AgentInfo . \ +?contributor mine:name ?contributorName . \ +?contributor mine:type ?contributorType . \ +?record mine:originalId ?originalId . \ + ?record mine:temporal ?t .\ + ?t a mine:AriadneTemporal . \ +?t mine:periodName ?temporalPeriodName . \ +?record mine:language ?language . \ +} \ +WHERE { \ +GRAPH { \ +?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . \ +?x rdfs:label ?publisherName . \ +?record aocat:has_native_subject ?ns . \ +?ns skos:prefLabel ?nativeSbj . \ +?type skos:prefLabel ?resourceType . \ +?record aocat:has_spatial_coverage ?spatial . \ +?spatial aocat:has_place_name ?spatialPlaceName . \ +?spatial aocat:has_latitude ?spatialLocationLat . \ +?spatial aocat:has_longitude ?spatialLocationLon . \ +?record aocat:has_access_policy ?accessPolicy . \ +?record aocat:has_landing_page ?z . \ +?z rdfs:label ?landingPage . \ +?record aocat:has_title ?title . \ +?record aocat:has_access_policy ?accessRights . \ +?record aocat:has_description ?description . \ +?record aocat:has_creator ?contributor . \ +?contributor rdfs:label ?contributorName . \ +?record aocat:has_original_id ?originalId . \ +?record aocat:has_temporal_coverage ?t . \ +?t aocat:has_period ?p . \ +?p skos:prefLabel ?temporalPeriodName . \ +?l skos:prefLabel ?language . \ +BIND("organization" as ?contributorType) . \ +BIND("organization" as ?publisherType) . \ +BIND(uri(concat('http://www.myprefix/location/', md5(concat(str(?spatialLocationLat), str(?spatialLocationLon))))) as ?blocation) \ +} \ +} + + +sparql.query3=PREFIX aocat: \ +PREFIX rdfs: \ +PREFIX skos: \ +PREFIX mine: \ +PREFIX onto: \ +CONSTRUCT {\ + ?record mine:resourcetype "AriadneCatalogEntry" .\ + ?record mine:identifier ?record .\ + ?record mine:publisher ?x .\ + ?x mine:type ?publisherType .\ +?record mine:issued ?issued .\ +?record mine:modified ?modified .\ +?record mine:accessPolicy ?accessPolicy .\ +?record mine:landingPage ?landingPage .\ +?record mine:title ?title .\ +?record mine:accessRights ?accessRights .\ +?record mine:description ?description .\ +?record mine:contributor ?contributor .\ +?record mine:originalId ?originalId .\ +}\ +WHERE {\ +?record aocat:has_access_policy ?accessPolicy .\ +?record aocat:has_landing_page ?z .\ +?z rdfs:label ?landingPage .\ +?record aocat:has_title ?title .\ +?record aocat:has_access_policy ?accessRights .\ +?record aocat:has_description ?description .\ +?record aocat:has_original_id ?originalId .\ +BIND("organization" as ?contributorType) .\ +BIND("organization" as ?publisherType) .\ +}\ +limit 20 + +catalog.entry.path=$[*][?(@['http://www.myprefix/resourcetype'][0]['value']=='AriadneCatalogEntry')] +general.classpath=ariadneplus.elasticsearch.model. +type.path=http://www.myprefix/resourcetype +exclude.predicates=["http://www.myprefix/resourcetype", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] +class.map.specifications={\ +"AriadneTemporal": {\ +"class_type": "prototype",\ +"mappings": {\ +"http://www.myprefix/from": {\ +"class_field": "From",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/periodName": {\ +"class_field": "PeriodName",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/until": {\ +"class_field": "Until",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/uri": {\ +"class_field": "Uri",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AriadneGeoPoint": {\ +"class_type": "unique",\ +"mappings": {\ +"http://www.myprefix/lat": {\ +"class_field": "Lat",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/lon": {\ +"class_field": "Lon",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"Spatial": {\ +"class_type": "prototype",\ +"mappings":{"http://www.myprefix/address": {\ +"class_field": "Address",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/coordinateSystem": {\ +"class_field": "CoordinateSystem",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/country": {\ +"class_field": "Country",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/location": {\ +"class_field": "Location",\ +"external_reference": "AriadneGeoPoint",\ +"substring": "no"\ +},\ +"http://www.myprefix/placeName": {\ +"class_field": "PlaceName",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}}\ +},\ +"ArcheologicalResourceType": {\ +"class_type": "unique",\ +"mappings": {\ +"http://www.myprefix/id": {\ +"class_field": "Id",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/name": {\ +"class_field": "Name",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AgentInfo": {\ +"class_type": "prototype",\ +"mappings": {\ +"http://www.myprefix/name": {\ +"class_field": "Name",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/type": {\ +"class_field": "Type",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/email": {\ +"class_field": "Email",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/phone": {\ +"class_field": "Phone",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"NativeSubject": {\ +"class_type": "prototype",\ +"mappings": {\ +"http://www.myprefix/prefLabel": {\ +"class_field": "PrefLabel",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/rdfAbout": {\ +"class_field": "RdfAbout",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +}\ +}\ +},\ +"AriadneCatalogEntry": {\ +"class_type": "unique",\ +"mappings": {\ +"http://www.myprefix/accessPolicy": {\ +"class_field": "AccessPolicy",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/identifier": {\ +"class_field": "Identifier",\ +"substring": "yes",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/publisher": {\ +"class_field": "Publisher",\ +"external_reference": "AgentInfo",\ +"substring": "no"\ +},\ +"http://www.myprefix/archeologicalResourceType": {\ +"class_field": "ArcheologicalResourceType",\ +"external_reference": "ArcheologicalResourceType",\ +"substring": "no"\ +},\ +"http://www.myprefix/issued": {\ +"class_field": "Issued",\ +"element_type": "java.lang.String",\ +"substring": "no"\ +},\ +"http://www.myprefix/resourceType": {\ +"class_field": "ResourceType",\ +"element_type": "java.lang.String",\ +"substring": "no"\ +},\ +"http://www.myprefix/modified": {\ +"class_field": "Modified",\ +"element_type": "java.lang.String",\ +"substring": "no"\ +},\ +"http://www.myprefix/nativeSubject": {\ +"class_field": "NativeSubject",\ +"substring": "no",\ +"external_reference": "NativeSubject"\ +},\ +"http://www.myprefix/spatial": {\ +"class_field": "Spatial",\ +"substring": "no",\ +"external_reference": "Spatial"\ +},\ +"http://www.myprefix/landingPage": {\ +"class_field": "LandingPage",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/title": {\ +"class_field": "Title",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/accessRights": {\ +"class_field": "AccessRights",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/description": {\ +"class_field": "Description",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/contributor": {\ +"class_field": "Contributor",\ +"substring": "no",\ +"external_reference": "AgentInfo"\ +},\ +"http://www.myprefix/originalId": {\ +"class_field": "OriginalId",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/temporal": {\ +"class_field": "Temporal",\ +"substring": "no",\ +"external_reference": "AriadneTemporal"\ +},\ +"http://www.myprefix/language": {\ +"class_field": "Language",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +},\ +"http://www.myprefix/partOf": {\ +"class_field": "PartOf",\ +"substring": "no",\ +"element_type": "java.lang.String"\ +}\ +}\ +}\ +} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java new file mode 100644 index 0000000..ae7cd20 --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java @@ -0,0 +1,40 @@ +/** + * + */ +package eu.dnetlib.ariadneplus; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.springframework.beans.factory.annotation.Autowired; + +import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; +import eu.dnetlib.ariadneplus.reader.ResourceManager; +import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService; +import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON; + +/** + * @author enrico.ottonello + * + */ +@RunWith(JUnit4.class) +public class GraphDbReaderTest { + @Autowired + private RunSPARQLQueryService runSPQRLQuery; + @Autowired + private ParseRDFJSON parser; + @Autowired + private ResourceManager resourceManager; + @Autowired + private BulkUpload bulkUpload; + + @Test + public void readTest() throws Exception { + + runSPQRLQuery.setParser(parser); + runSPQRLQuery.setResourceManager(resourceManager); + runSPQRLQuery.setBulkUpload(bulkUpload); + runSPQRLQuery.executeQueryGraph(); + + } +}