D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added graphDB reader and ES indexer

This commit is contained in:
Enrico Ottonello 2020-06-10 19:39:53 +02:00
parent fa1d268b36
commit b80274fff3
17 changed files with 890 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java
View File

@ -5,6 +5,9 @@ import java.lang.reflect.InvocationTargetException;
import javax.annotation.PostConstruct;
import eu.dnetlib.ariadneplus.elasticsearch.model.AgentInfo;
import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
import org.apache.http.HttpHost;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
@ -16,9 +19,6 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
@Service
public class BulkUpload {
@ -28,22 +28,33 @@ public class BulkUpload {
private RestHighLevelClient client;
@PostConstruct
private void init() throws IOException {
public void init() throws IOException {
client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("localhost",9200,"http")));
new HttpHost("elastic-test.ariadne.d4science.org",9200,"http")));
}
public void index(ResourceManager manager) {
BulkRequest request = new BulkRequest();
while (manager.hasNext()){
try {
AriadneCatalogEntry ace = ((AriadneCatalogEntry) manager.next());
request.add(new IndexRequest("prova_via_code").id(ace.getOriginalId())
Object next = manager.next();
AriadneCatalogEntry ace = ((AriadneCatalogEntry) next);
AgentInfo testPublisher = new AgentInfo();
testPublisher.setName("TEST");
ace.getPublisher().add(testPublisher);
String[] splits = ace.getIdentifier().split("/");
request.add(new IndexRequest("catalog_test").id(splits[splits.length-1])
.source(ace.toJson(),XContentType.JSON));
System.out.println("indexing to ES record "+ace.getOriginalId());
System.out.println("indexing to ES record "+ace.toJson());
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
System.out.println("indexing to ES record completed "+bulkResponse.status());
if (bulkResponse.hasFailures()) {
System.out.println("FailureMessage: "+bulkResponse.buildFailureMessage());
}
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {

8
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java
View File

@ -55,16 +55,16 @@ public class AgentInfo {
AgentInfo pi = new AgentInfo();
for (Map.Entry<String, JsonElement> entry : json.getAsJsonObject().entrySet()){
switch (entry.getKey()){
case "http://www.myprefix/name" :
case "https://www.ariadne-infrastructure.eu/property/name" :
pi.setName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/type":
case "https://www.ariadne-infrastructure.eu/property/type":
pi.setType(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/email":
case "https://www.ariadne-infrastructure.eu/property/email":
pi.setEmail(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/phone":
case "https://www.ariadne-infrastructure.eu/property/phone":
pi.setPhone(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
}

4
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/ArcheologicalResourceType.java
View File

@ -16,10 +16,10 @@ public class ArcheologicalResourceType {
ArcheologicalResourceType art = new ArcheologicalResourceType();
for (Map.Entry<String, JsonElement> entry : json.getAsJsonObject().entrySet()){
switch (entry.getKey()){
case "http://www.myprefix/id" :
case "https://www.ariadne-infrastructure.eu/property/id" :
art.setId(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/name":
case "https://www.ariadne-infrastructure.eu/property/name":
String tmp = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString();
art.setName(tmp);
break;

63
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java
View File

@ -1,8 +1,9 @@
package eu.dnetlib.ariadneplus.elasticsearch.model;
import com.google.gson.Gson;
import java.util.List;
import eu.dnetlib.ariadneplus.reader.utils.ESUtils;
import java.util.List;
public class AriadneCatalogEntry {
private List<AatSubject> aatSubjects;
@ -21,7 +22,7 @@ public class AriadneCatalogEntry {
private List<ItemMetadataStructure> hasItemMetadataStructure;
private List<MetadataRecord> hasMetadataRecord;
private String identifier;
private String partOf;
private String isPartOf;
private String issued;
private List<String> keyword;
private String landingPage;
@ -40,6 +41,8 @@ public class AriadneCatalogEntry {
private String rights;
private List<AgentInfo> scientificResponsible;
private List<Spatial> spatial;
// private List<SpatialRegion> spatialRegion;
// private List<SpatialRegionPoint> spatialRegionPoint;
private List<AgentInfo> technicalResponsible;
private List<AriadneTemporal> temporal;
@ -173,12 +176,12 @@ public class AriadneCatalogEntry {
this.identifier = identifier;
}
public String getPartOf() {
return partOf;
public String getIsPartOf() {
return isPartOf;
}
public void setPartOf(String partOf) {
this.partOf = partOf;
public void setIsPartOf(String isPartOf) {
this.isPartOf = isPartOf;
}
public String getIssued() {
@ -186,7 +189,7 @@ public class AriadneCatalogEntry {
}
public void setIssued(String issued) {
this.issued = issued;
this.issued = ESUtils.getESFormatDate(issued);
}
public List<String> getKeyword() {
@ -226,7 +229,7 @@ public class AriadneCatalogEntry {
}
public void setModified(String modified) {
this.modified = modified;
this.modified = ESUtils.getESFormatDate(modified);
}
public List<NativeSubject> getNativeSubject() {
@ -322,7 +325,12 @@ public class AriadneCatalogEntry {
}
public void setSpatial(List<Spatial> spatial) {
this.spatial = spatial;
if (this.spatial==null) {
this.spatial = spatial;
}
else {
this.spatial.addAll(spatial);
}
}
public List<AgentInfo> getTechnicalResponsible() {
@ -357,13 +365,13 @@ public class AriadneCatalogEntry {
// JsonObject content = json.getAsJsonObject();
// for (Map.Entry<String, JsonElement> stringJsonElementEntry : content.entrySet()) {
// switch (stringJsonElementEntry.getKey()){
// case "http://www.myprefix/accessPolicy":
// case "https://www.ariadne-infrastructure.eu/property/accessPolicy":
// acim.setAccessPolicy(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/accessRights":
// case "https://www.ariadne-infrastructure.eu/property/accessRights":
// acim.setAccessRights(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/contributor":
// case "https://www.ariadne-infrastructure.eu/property/contributor":
// JsonArray contributor_array = stringJsonElementEntry.getValue().getAsJsonArray();
// List<AgentInfo> contributor_list = new ArrayList();
// for (int i = 0; i < contributor_array.size() ; i++ ){
@ -373,25 +381,25 @@ public class AriadneCatalogEntry {
// }
// acim.setContributor(contributor_list);
// break;
// case "http://www.myprefix/description":
// case "https://www.ariadne-infrastructure.eu/property/description":
// acim.setDescription(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/isPartOf":
// case "https://www.ariadne-infrastructure.eu/property/isPartOf":
// acim.setPartOf(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/issued":
// case "https://www.ariadne-infrastructure.eu/property/issued":
// acim.setIssued(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/landingPage":
// case "https://www.ariadne-infrastructure.eu/property/landingPage":
// acim.setLandingPage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/language":
// case "https://www.ariadne-infrastructure.eu/property/language":
// acim.setLanguage(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/modified":
// case "https://www.ariadne-infrastructure.eu/property/modified":
// acim.setModified(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/nativeSubject":
// case "https://www.ariadne-infrastructure.eu/property/nativeSubject":
// JsonArray nativeSubject_array = stringJsonElementEntry.getValue().getAsJsonArray();
// List<NativeSubject> nativeSubject_list = new ArrayList();
// for (int i = 0; i < nativeSubject_array.size() ; i++ ){
@ -401,13 +409,13 @@ public class AriadneCatalogEntry {
// }
// acim.setNativeSubject(nativeSubject_list);
// break;
// case "http://www.myprefix/originalId":
// case "https://www.ariadne-infrastructure.eu/property/originalId":
// acim.setOriginalId(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/resourceType":
// case "https://www.ariadne-infrastructure.eu/property/resourceType":
// acim.setResourceType(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/spatial":
// case "https://www.ariadne-infrastructure.eu/property/spatial":
// JsonArray spatial_array = stringJsonElementEntry.getValue().getAsJsonArray();
// List<Spatial> spatial_list = new ArrayList();
// for (int i = 0; i < spatial_array.size() ; i++ ){
@ -417,7 +425,7 @@ public class AriadneCatalogEntry {
// }
// acim.setSpatial(spatial_list);
// break;
// case "http://www.myprefix/temporal":
// case "https://www.ariadne-infrastructure.eu/property/temporal":
// JsonArray temporal_array = stringJsonElementEntry.getValue().getAsJsonArray();
// List<AriadneTemporal> temporal_list = new ArrayList<>();
// for(int i=0; i < temporal_array.size(); i++){
@ -426,10 +434,10 @@ public class AriadneCatalogEntry {
// }
// acim.setTemporal(temporal_list);
// break;
// case "http://www.myprefix/title":
// case "https://www.ariadne-infrastructure.eu/property/title":
// acim.setTitle(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "http://www.myprefix/publisher":
// case "https://www.ariadne-infrastructure.eu/property/publisher":
// JsonArray publisher_array = stringJsonElementEntry.getValue().getAsJsonArray();
// List<AgentInfo> publisher_list = new ArrayList();
// for (int i = 0; i < publisher_array.size() ; i++ ){
@ -439,7 +447,7 @@ public class AriadneCatalogEntry {
// }
// acim.setPublisher(publisher_list);
// break;
// case "http://www.myprefix/archeologicalResourceType":
// case "https://www.ariadne-infrastructure.eu/property/archeologicalResourceType":
// acim.setArcheologicalResourceType(ArcheologicalResourceType.fromRDFJson(map.get(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString())));
// }
//
@ -449,5 +457,4 @@ public class AriadneCatalogEntry {
// System.out.println(acim.toJson());
// return acim;
// }
}
//https://ariadne-infrastructure.eu/aocat
}

4
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java
View File

@ -36,10 +36,10 @@ public class AriadneGeoPoint {
AriadneGeoPoint agp = new AriadneGeoPoint();
for (Map.Entry<String, JsonElement> stringJsonElementEntry : json.getAsJsonObject().entrySet()) {
switch (stringJsonElementEntry.getKey()){
case "http://www.myprefix/lat":
case "https://www.ariadne-infrastructure.eu/property/lat":
agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/lon":
case "https://www.ariadne-infrastructure.eu/property/lon":
agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
}

8
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneTemporal.java
View File

@ -15,16 +15,16 @@ public class AriadneTemporal {
AriadneTemporal at = new AriadneTemporal();
for (Map.Entry<String, JsonElement> entry : json.getAsJsonObject().entrySet()){
switch (entry.getKey()){
case "http://www.myprefix/from" :
case "https://www.ariadne-infrastructure.eu/property/from" :
at.setFrom(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/periodName":
case "https://www.ariadne-infrastructure.eu/property/periodName":
at.setPeriodName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/until":
case "https://www.ariadne-infrastructure.eu/property/until":
at.setUntil(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/uri":
case "https://www.ariadne-infrastructure.eu/property/uri":
at.setUri(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;

4
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/NativeSubject.java
View File

@ -38,10 +38,10 @@ public class NativeSubject {
NativeSubject pi = new NativeSubject();
for (Map.Entry<String, JsonElement> entry : json.getAsJsonObject().entrySet()){
switch (entry.getKey()){
case "http://www.myprefix/prefLabel" :
case "https://www.ariadne-infrastructure.eu/property/prefLabel" :
pi.setPrefLabel(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/rdfAbout":
case "https://www.ariadne-infrastructure.eu/property/rdfAbout":
pi.setRdfAbout(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;

54
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java
View File

@ -16,44 +16,8 @@ public class Spatial {
private String country;
private AriadneGeoPoint location;
private String placeName;
public static Spatial fromRDFJson(JsonElement json, Map<String,JsonElement> map) {
Spatial pi = new Spatial();
for (Map.Entry<String, JsonElement> entry : json.getAsJsonObject().entrySet()){
switch (entry.getKey()){
case "http://www.myprefix/address" :
pi.setAddress(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/boundingBoxMaxLat":
pi.setBoundingBoxMaxLat(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/boundingBoxMaxLon":
pi.setBoundingBoxMaxLon(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/boundingBoxMinLat":
pi.setBoundingBoxMinLat(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/boundingBoxMinLon":
pi.setBoundingBoxMinLon(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/country":
pi.setCountry(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/coordinateSystem":
pi.setCoordinateSystem(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "http://www.myprefix/location":
String map_key = entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString();
pi.setLocation(AriadneGeoPoint.fromRDFJson(map.get(map_key)));
break;
case "http://www.myprefix/placeName":
pi.setPlaceName(entry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
}
}
return pi;
}
private String lat;
private String lon;
public String getAddress() {
return address;
@ -127,6 +91,20 @@ public class Spatial {
this.placeName = placeName;
}
public void setLat(String lat) {
if (this.getLocation()==null) {
this.setLocation(new AriadneGeoPoint());
}
this.getLocation().setLat(lat);
}
public void setLon(String lon) {
if (this.getLocation()==null) {
this.setLocation(new AriadneGeoPoint());
}
this.getLocation().setLon(lon);
}
public Spatial() {
}

33
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java
View File

@ -10,16 +10,16 @@ import java.util.Set;
import javax.annotation.PostConstruct;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
import eu.dnetlib.ariadneplus.reader.utils.ClassSpec;
import eu.dnetlib.ariadneplus.reader.utils.Mappings;
import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
import eu.dnetlib.ariadneplus.reader.utils.ClassSpec;
import eu.dnetlib.ariadneplus.reader.utils.Mappings;
import eu.dnetlib.ariadneplus.reader.utils.PropertiesMap;
import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
@ -32,11 +32,23 @@ public class ResourceManager {
private String general_classpath;
@Value("${exclude.predicates:[]}")
private String exclude_predicates;
@Value("${class.map.specifications:undefined}")
private String spec;
private List<String> not_parsable;
private ParseRDFJSON parser;
private PropertiesMap propertiesMap;
public void setup(String type_path, String general_classpath, String exclude_predicates, String spec) {
this.type_path = type_path;
this.general_classpath = general_classpath;
this.exclude_predicates = exclude_predicates;
propertiesMap = new PropertiesMap();
propertiesMap.fill(spec);
}
@PostConstruct
public void init(){
Type listType = new TypeToken<ArrayList<String>>(){}.getType();
@ -69,11 +81,20 @@ public class ResourceManager {
if(entry instanceof LinkedHashMap){
LinkedHashMap tmp = (LinkedHashMap)((JSONArray)((LinkedHashMap)entry).get(type_path)).get(0);
class_name = (String)tmp.get("value");
if (class_name.equals("provided record")) {
class_name = "AriadneCatalogEntry";
}
}
}
if (entry == null) {
System.out.println("entry NULL " );
return null;
}
System.out.println("class_name: " + class_name);
Class<?> c = Class.forName(general_classpath + class_name);
Object class_instance = c.newInstance();
ClassSpec class_spec = PropertiesMap.get(class_name);
ClassSpec class_spec = propertiesMap.get(class_name);
Set<?> keySet;
if(entry instanceof LinkedHashMap)
@ -101,7 +122,7 @@ public class ResourceManager {
setField.invoke(class_instance, getFieldValue(values.get(0)));
}
else{
if(PropertiesMap.get(map.getExternal_reference()).getClass_type().equals("prototype")){
if(propertiesMap.get(map.getExternal_reference()).getClass_type().equals("prototype")){
List<Object> value_list = new ArrayList<>();
for(Object value: values){
value_list.add(manage(ParseRDFJSON.get(getFieldValue(value)), map.getExternal_reference()));

304
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java
View File

@ -1,13 +1,10 @@
package eu.dnetlib.ariadneplus.reader;
import java.io.StringWriter;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Literal;
import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.model.Resource;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.query.GraphQuery;
import org.eclipse.rdf4j.query.GraphQueryResult;
import org.eclipse.rdf4j.query.QueryLanguage;
@ -21,14 +18,14 @@ import org.eclipse.rdf4j.rio.Rio;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
import java.io.StringWriter;
import java.util.Calendar;
@Service
public class RunSPARQLQueryService {
@Value("${sparql.query3:undefined}")
private String query3;
@Value("${sparql.query:undefined}")
private String query;
@Value("${repository.url:undefined}")
private String repository_url;
@ -41,12 +38,23 @@ public class RunSPARQLQueryService {
private ResourceManager resourceManager;
private BulkUpload bulkUpload;
private void openConnection(){
manager = new RemoteRepositoryManager(repository_url);
private static String username = null;
private static String pwd = null;
private static String graphDBUrl = null;
private static String graphDBRepository = null;
public void setupConnection(String username, String pwd, String graphDbUrl, String graphDbRepository) {
setUsername(username);
setPwd(pwd);
setGraphDBUrl(graphDbUrl);
setGraphDBRepository(graphDbRepository);
}
private void openConnection(){
manager = new RemoteRepositoryManager(getGraphDBUrl());
manager.init();
manager.setUsernameAndPassword("writer", "Writer01");
// repository = manager.getRepository("test01");
repository = manager.getRepository("ariadneplus-ts01");
manager.setUsernameAndPassword(getUsername(), getPwd());
repository = manager.getRepository(getGraphDBRepository());
connection = repository.getConnection();
}
@ -58,58 +66,213 @@ public class RunSPARQLQueryService {
}
public String executeQueryGraph(){
if (query3.equals("undefined"))
query = "PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/> \n" +
"PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> \n" +
"PREFIX skos: <http://www.w3.org/2004/02/skos/core#> \n" +
"PREFIX aoprop: <https://www.ariadne-infrastructure.eu/property/> \n" +
"PREFIX onto: <http://www.ontotext.com/>\n" +
"PREFIX ariadneplus: <https://ariadne-infrastructure.eu/aocat/>\n" +
"PREFIX time: <http://www.w3.org/2006/time#>\n" +
"\n" +
"CONSTRUCT { \n" +
"%s aoprop:identifier %s . \n" +
"%s aoprop:originalId ?originalId . \n" +
"%s aoprop:issued ?issued .\n" +
"%s aoprop:modified ?modified .\n" +
"%s aoprop:partOf ?partOf .\n" +
"%s aoprop:creator ?creator . \n" +
"?creator aoprop:name ?creatorName .\n" +
"?creator aoprop:email ?creatorEmail . \n" +
"%s aoprop:contributor ?contributor . \n" +
"?contributor aoprop:name ?contributorName .\n" +
"?contributor aoprop:email ?contributorEmail .\n" +
"%s aoprop:legalResponsible ?legalResponsible . \n" +
"?legalResponsible aoprop:name ?legalResponsibleName .\n" +
"?legalResponsible aoprop:email ?legalResponsibleEmail . \n" +
"%s aoprop:owner ?owner . \n" +
"?owner aoprop:name ?ownerName .\n" +
"?owner aoprop:email ?ownerEmail . \n" +
"%s aoprop:publisher ?publisher . \n" +
"?publisher aoprop:name ?publisherName . \n" +
"?publisher aoprop:email ?publisherEmail . \n" +
"%s aoprop:accessPolicy ?accessPolicy . \n" +
"%s aoprop:accessRights ?accessRights .\n" +
"%s aoprop:landingPage ?landingPage .\n" +
"%s aoprop:spatialRegion ?spatialRegion . \n" +
"?spatialRegion aoprop:placeName ?spatialPlaceName .\n" +
"?spatialRegion aoprop:spatialCoordinateSystem ?spatialCoordinateSystem . \n" +
"%s aoprop:spatialRegionPoint ?spatialRegionPoint . \n" +
"?spatialRegionPoint aoprop:lat ?spatialLocationLat . \n" +
"?spatialRegionPoint aoprop:lon ?spatialLocationLon .\n" +
"%s aoprop:spatialRegionBox ?spatialRegionBox . \n" +
"?spatialRegionBox aoprop:boxMaxLat ?spatialLocationBBMaxLat .\n" +
"?spatialRegionBox aoprop:boxMaxLon ?spatialLocationBBMaxLon .\n" +
"?spatialRegionBox aoprop:boxMinLat ?spatialLocationBBMinLat .\n" +
"?spatialRegionBox aoprop:boxMinLon ?spatialLocationBBMinLon .\n" +
"%s aoprop:uri ?temporal .\n" +
"?temporal aoprop:periodName ?temporalPeriodName .\n" +
"?temporal aoprop:from ?temporalFrom .\n" +
"?temporal aoprop:until ?temporalUntil .\n" +
"%s aoprop:uri ?temporalNative .\n" +
"?temporalNative aoprop:periodName ?temporalNativePeriodName .\n" +
"%s aoprop:archeologicalResourceType ?archeologicalResourceType . \n" +
"?archeologicalResourceType aoprop:name ?archeologicalResourceTypeName . \n" +
"%s aoprop:resourceType ?resourceType . \n" +
"%s aoprop:nativeSubject ?nativeSubject . \n" +
"?nativeSubject aoprop:prefLabel ?nativeSubjectPrefLabel .\n" +
"?nativeSubject aoprop:rdfAbout ?nativeSubject .\n" +
"%s aoprop:derivedSubject ?derivedSubject .\n" +
"?derivedSubject aoprop:prefLabel ?derivedSubjectPrefLabel .\n" +
"?derivedSubject aoprop:source \"Getty AAT\" .\n" +
"%s aoprop:aatSubjects ?derivedSubject .\n" +
"?derivedSubject aoprop:id ?derivedSubject . \n" +
"?derivedSubject aoprop:label ?derivedSubjectPrefLabel .\n" +
"?derivedSubject aoprop:lang ?aatSubjectsLang .\n" +
"%s aoprop:title ?title . \n" +
"%s aoprop:description ?description . \n" +
"%s aoprop:language ?language . \n" +
"}\n" +
"from <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::aat>\n" +
"from <https://ariadne-infrastructure.eu/ariadneplus::ads::aatplus>\n" +
"from <https://ariadne-infrastructure.eu/ariadneplus::ads::periodo>\n" +
"from <https://ariadne-infrastructure.eu/ariadneplus::ads::periodoplus>\n" +
"from <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::271>\n" +
"where { \n" +
" \t%s\taocat:has_language / skos:prefLabel ?language .\n" +
" \t%s\taocat:has_original_id ?originalId .\n" +
" \t%s aocat:is_part_of ?partOf .\n" +
" \t%s\taocat:has_creator ?creator .\n" +
" ?creator aocat:has_name ?creatorName .\n" +
"\t %s aocat:has_title ?title .\n" +
" \t%s aocat:has_type / skos:prefLabel ?resourceType .\t\n" +
" \t%s aocat:has_native_subject ?nativeSubject .\n" +
" ?nativeSubject skos:prefLabel ?nativeSubjectPrefLabel .\n" +
" optional {\n" +
"\t\t\t%s aocat:has_derived_subject ?derivedSubject .\n" +
" \t?derivedSubject skos:prefLabel ?derivedSubjectPrefLabel .\n" +
" }\n" +
" optional {\n" +
"\t ?creator aocat:has_email ?creatorEmail .\n" +
" \t}\n" +
" optional {\n" +
" %s aocat:has_description ?description .\n" +
" }\n" +
" optional {\n" +
" %s aocat:has_access_policy / rdfs:label ?accessPolicy . \n" +
" }\n" +
" optional {\n" +
" \t%s aocat:has_landing_page / rdfs:label ?landingPage .\n" +
" }\n" +
" \toptional {\n" +
" \t%s aocat:has_temporal_coverage ?temporalNative .\n" +
" \t?temporalNative aocat:has_native_period / skos:prefLabel ?temporalNativePeriodName .\n" +
" }\n" +
" optional {\n" +
" \t%s aocat:has_temporal_coverage ?temporal .\n" +
" \t?temporal aocat:has_period / skos:prefLabel ?temporalPeriodName .\n" +
" \toptional {\n" +
" ?temporal aocat:from ?temporalFrom .\n" +
" ?temporal aocat:until ?temporalUntil .\n" +
" \t}\n" +
" }\n" +
" \t\n" +
" \t{ \n" +
"\t\tselect * \n" +
" where {\n" +
" %s aocat:is_part_of ?collection .\n" +
" %s aocat:was_issued ?issued .\n" +
" %s aocat:was_modified ?modified .\n" +
" %s aocat:has_contributor ?contributor .\n" +
" ?contributor aocat:has_name ?contributorName .\n" +
" %s aocat:has_responsible ?legalResponsible .\n" +
" ?legalResponsible aocat:has_name ?legalResponsibleName .\n" +
" %s aocat:has_owner ?owner .\n" +
" ?owner aocat:has_name ?ownerName .\n" +
" %s aocat:has_publisher ?publisher . \n" +
" ?publisher aocat:has_name ?publisherName .\n" +
" %s aocat:has_access_rights ?accessRights .\n" +
" %s aocat:has_ARIADNE_subject ?archeologicalResourceType . \n" +
" ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .\n" +
" optional {\n" +
" ?contributor aocat:has_email ?contributorEmail .\n" +
" }\n" +
" optional {\n" +
" ?legalResponsible aocat:has_email ?legalResponsibleEmail .\n" +
" }\n" +
" optional {\n" +
" ?owner aocat:has_email ?ownerEmail . \n" +
" }\n" +
" optional {\n" +
" ?publisher aocat:has_email ?publisherEmail .\n" +
" }\n" +
" \t}\n" +
" \t}\n" +
" \n" +
" \t{\n" +
" select * \n" +
" where {\n" +
" \t%s\taocat:has_spatial_coverage ?spatialRegion .\n" +
" ?spatialRegion aocat:has_place_name ?spatialPlaceName .\n" +
" optional {\n" +
" ?spatialRegion aocat:has_coordinate_system ?spatialCoordinateSystem .\n" +
" }\n" +
" optional {\n" +
" %s\taocat:has_spatial_coverage ?spatialRegionPoint .\n" +
" ?spatialRegionPoint aocat:has_latitude ?spatialLocationLat ; \n" +
" \t\t aocat:has_longitude ?spatialLocationLon . \n" +
" }\n" +
" optional {\n" +
" %s\taocat:has_spatial_coverage ?spatialRegionBox .\n" +
" ?spatialRegionBox aocat:has_bounding_box_max_lat ?spatialLocationBBMaxLat ; \n" +
" \t\t aocat:has_bounding_box_max_lon ?spatialLocationBBMaxLon ;\n" +
" \t\t aocat:has_bounding_box_min_lat ?spatialLocationBBMinLat ;\n" +
" \t aocat:has_bounding_box_min_lon ?spatialLocationBBMinLon ;\n" +
" }\n" +
" }\t\n" +
" \t}\n" +
" BIND (lang(?derivedSubjectPrefLabel) as ?aatSubjectsLang)\n" +
"}";
if (query.equals("undefined"))
return null;
openConnection();
query = query.replaceAll("%s", "<https://ariadne-infrastructure.eu/aocat/Resource/02E4F4B5-24B7-3AD7-B460-CFA8B1F0BD1F>");
openConnection();
StringWriter recordWriter = null;
Model model = null;
Model resultsModel = null;
String jsonRecord = null;
try {
GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query3);
System.out.println("Start connection Time: "+Calendar.getInstance().getTime().toString());
GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query);
GraphQueryResult graphQueryResult = graphQuery.evaluate();
System.out.println("as model ...");
model = QueryResults.asModel(graphQueryResult);
resultsModel = QueryResults.asModel(graphQueryResult);
graphQueryResult.close();
int resourceCount = 0;
System.out.println("counting resources ... " );
ValueFactory factory = SimpleValueFactory.getInstance();
IRI iri = factory.createIRI("http://www.myprefix/resourcetype");
Literal value = factory.createLiteral("AriadneCatalogEntry");
for (Resource record: model.filter(null, iri, value).subjects()) {
System.out.println();
Model recordModel = null;
// RDFWriter rdfRecordWriter = null;
resourceCount+=1;
// if (resourceCount==12) {
// break;
// }
recordModel = model.filter(record, null, null);
if (recordModel!=null && !recordModel.isEmpty()) {
recordWriter = new StringWriter();
RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter);
Rio.write(recordModel, rdfRecordWriter);
// System.out.println("record json: "+ recordWriter.toString());
parser.parse(recordWriter.toString());
resourceManager.manage(parser);
bulkUpload.index(resourceManager);
}
}
System.out.println();
System.out.println("count resources: "+ resourceCount);
System.out.println("count statements: " + model.size());
System.out.println("index to Elastic Search completed");
}catch(Exception e){
System.out.println("End connection Time: "+Calendar.getInstance().getTime().toString());
System.out.println("count statements: " + resultsModel.size());
recordWriter = new StringWriter();
RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter);
Rio.write(resultsModel, rdfRecordWriter);
System.out.println("RDF > json record: "+recordWriter.toString());
parser.parse(recordWriter.toString());
resourceManager.manage(parser);
bulkUpload.index(resourceManager);
} catch(Exception e){
e.printStackTrace();
}finally{
} finally{
closeConnection();
if (model!=null) {
model.clear();
if (resultsModel!=null) {
resultsModel.clear();
}
}
return jsonRecord;
}
private void dumpModel(Model model) {
System.out.print(" [ dump model ] ");
for (Statement stmt: model) {
System.out.println(stmt.toString());
}
}
public ParseRDFJSON getParser() {
return parser;
}
@ -139,4 +302,35 @@ public class RunSPARQLQueryService {
this.bulkUpload = bulkUpload;
}
public static String getUsername() {
return username;
}
public static String getPwd() {
return pwd;
}
public static String getGraphDBUrl() {
return graphDBUrl;
}
public static String getGraphDBRepository() {
return graphDBRepository;
}
public static void setUsername(String username) {
RunSPARQLQueryService.username = username;
}
public static void setPwd(String pwd) {
RunSPARQLQueryService.pwd = pwd;
}
public static void setGraphDBUrl(String graphDBUrl) {
RunSPARQLQueryService.graphDBUrl = graphDBUrl;
}
public static void setGraphDBRepository(String graphDBRepository) {
RunSPARQLQueryService.graphDBRepository = graphDBRepository;
}
}

11
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/json/ParseRDFJSON.java
View File

@ -20,8 +20,7 @@ public class ParseRDFJSON {
static JSONObject map ;
@Value("${catalog.entry.path:undefined}")
private String query;
private String catalogEntryJsonPath;
private String json;
@ -45,7 +44,7 @@ public class ParseRDFJSON {
setJson(json);
fillMap();
DocumentContext jsonContext = JsonPath.parse(json);
JSONArray entries = jsonContext.read(query);
JSONArray entries = jsonContext.read(getCatalogEntryJsonPath());
int size = entries.size();
it = entries.iterator();
}
@ -62,5 +61,11 @@ public class ParseRDFJSON {
return (JSONObject) map.get(key);
}
public String getCatalogEntryJsonPath() {
return catalogEntryJsonPath;
}
public void setCatalogEntryJsonPath(String catalogEntryJsonPath) {
this.catalogEntryJsonPath = catalogEntryJsonPath;
}
}

14
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java Normal file
View File

@ -0,0 +1,14 @@
package eu.dnetlib.ariadneplus.reader.utils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
public class ESUtils {
private static DateTimeFormatter originalRecordDateFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy");
private static DateTimeFormatter elasticSearchDateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
public static String getESFormatDate(String originalDate) {
LocalDate parsedDate = LocalDate.parse(originalDate, originalRecordDateFormatter);
return parsedDate.format(elasticSearchDateFormatter);
}
}

22
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/PropertiesMap.java
View File

@ -1,31 +1,29 @@
package eu.dnetlib.ariadneplus.reader.utils;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
@Component
public class PropertiesMap {
@Value("${class.map.specifications:undefined}")
private String spec;
// @Value("${class.map.specifications:undefined}")
// private String spec;
private static Map<String, ClassSpec> map;
public static ClassSpec get(String key){
public ClassSpec get(String key){
return map.get(key);
}
@PostConstruct
public void fill(){
public void fill(String spec){
Type mapType = new TypeToken<HashMap<String,ClassSpec>>(){}.getType();
map = new Gson().fromJson(spec,mapType);

277
dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties
View File

@ -5,147 +5,38 @@ server.port=8281
graphdb.serverUrl=http://localhost:7200/
graphdb.writer.user=writer
graphdb.writer.pwd=writer01
graphdb.repository=ariadneprova
graphdb.writer.pwd=****
graphdb.repository=ariadneplus-ts01
graphdb.sparqlUrl = http://localhost:7200/sparql
graphdb.baseURI=https://ariadne-infrastructure.eu/
#repository.url=http://localhost:7200
repository.url=http://graphdb-test.ariadne.d4science.org:7200
sparql.query2=PREFIX skos: <http://www.w3.org/2004/02/skos/core#> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/> PREFIX mine: <http://www.myprefix/> CONSTRUCT { ?archeologicalResourceType a mine:archeologicalResourceType . ?record mine:archeologicalResourceType ?archeologicalResourceType . ?archeologicalResourceType mine:name ?archeologicalResourceTypeName . } WHERE { GRAPH <api_________::ariadne_plus::ads::sample_partition_1> { ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . } }
sparql.query=PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/> \
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> \
PREFIX skos: <http://www.w3.org/2004/02/skos/core#> \
PREFIX mine: <http://www.myprefix/> \
CONSTRUCT { \
?record mine:resourcetype "AriadneCatalogEntry" . \
?record mine:identifier ?record .\
?record mine:publisher ?x . \
?x a mine:AgentInfo . \
?x mine:name ?publisherName . \
?x mine:type ?publisherType . \
?archeologicalResourceType a mine:archeologicalResourceType . \
?record mine:archeologicalResourceType ?archeologicalResourceType . \
?archeologicalResourceType mine:name ?archeologicalResourceTypeName . \
?record mine:issued ?issued . \
?record mine:resourceType ?resourceType . \
?record mine:modified ?modified . \
?record mine:nativeSubject ?ns . \
?ns a mine:NativeSubject . \
?ns mine:prefLabel ?nativeSbj . \
?spatial a mine:Spatial . \
?record mine:spatial ?spatial . \
?spatial mine:placeName ?spatialPlaceName . \
?spatial mine:location ?blocation . \
?blocation a mine:AriadneGeoPoint . \
?blocation mine:lat ?spatialLocationLat . \
?blocation mine:lon ?spatialLocationLon . \
?record mine:accessPolicy ?accessPolicy . \
?record mine:landingPage ?landingPage . \
?record mine:title ?title . \
?record mine:accessRights ?accessRights . \
?record mine:description ?description . \
?record mine:contributor ?contributor . \
?contributor a mine:AgentInfo . \
?contributor mine:name ?contributorName . \
?contributor mine:type ?contributorType . \
?record mine:originalId ?originalId . \
?record mine:temporal ?t .\
?t a mine:AriadneTemporal . \
?t mine:periodName ?temporalPeriodName . \
?record mine:language ?language . \
} \
WHERE { \
GRAPH <api_________::ariadne_plus::ads::sample_partition_1> { \
?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . \
?x rdfs:label ?publisherName . \
?record aocat:has_native_subject ?ns . \
?ns skos:prefLabel ?nativeSbj . \
?type skos:prefLabel ?resourceType . \
?record aocat:has_spatial_coverage ?spatial . \
?spatial aocat:has_place_name ?spatialPlaceName . \
?spatial aocat:has_latitude ?spatialLocationLat . \
?spatial aocat:has_longitude ?spatialLocationLon . \
?record aocat:has_access_policy ?accessPolicy . \
?record aocat:has_landing_page ?z . \
?z rdfs:label ?landingPage . \
?record aocat:has_title ?title . \
?record aocat:has_access_policy ?accessRights . \
?record aocat:has_description ?description . \
?record aocat:has_creator ?contributor . \
?contributor rdfs:label ?contributorName . \
?record aocat:has_original_id ?originalId . \
?record aocat:has_temporal_coverage ?t . \
?t aocat:has_period ?p . \
?p skos:prefLabel ?temporalPeriodName . \
?l skos:prefLabel ?language . \
BIND("organization" as ?contributorType) . \
BIND("organization" as ?publisherType) . \
BIND(uri(concat('http://www.myprefix/location/', md5(concat(str(?spatialLocationLat), str(?spatialLocationLon))))) as ?blocation) \
} \
}
sparql.query3=PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>\
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>\
PREFIX mine: <http://www.myprefix/>\
PREFIX onto: <http://www.ontotext.com/>\
CONSTRUCT {\
?record mine:resourcetype "AriadneCatalogEntry" .\
?record mine:identifier ?record .\
?record mine:publisher ?x .\
?x mine:type ?publisherType .\
?record mine:issued ?issued .\
?record mine:modified ?modified .\
?record mine:accessPolicy ?accessPolicy .\
?record mine:landingPage ?landingPage .\
?record mine:title ?title .\
?record mine:accessRights ?accessRights .\
?record mine:description ?description .\
?record mine:contributor ?contributor .\
?record mine:originalId ?originalId .\
}\
WHERE {\
?record aocat:has_access_policy ?accessPolicy .\
?record aocat:has_landing_page ?z .\
?z rdfs:label ?landingPage .\
?record aocat:has_title ?title .\
?record aocat:has_access_policy ?accessRights .\
?record aocat:has_description ?description .\
?record aocat:has_original_id ?originalId .\
BIND("organization" as ?contributorType) .\
BIND("organization" as ?publisherType) .\
}\
limit 20
catalog.entry.path=$[*][?(@['http://www.myprefix/resourcetype'][0]['value']=='AriadneCatalogEntry')]
general.classpath=ariadneplus.elasticsearch.model.
type.path=http://www.myprefix/resourcetype
exclude.predicates=["http://www.myprefix/resourcetype", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"]
catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='provided record')]
general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model.
type.path=https://www.ariadne-infrastructure.eu/property/resourceType
exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"]
class.map.specifications={\
"AriadneTemporal": {\
"class_type": "prototype",\
"mappings": {\
"http://www.myprefix/from": {\
"https://www.ariadne-infrastructure.eu/property/from": {\
"class_field": "From",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/periodName": {\
"https://www.ariadne-infrastructure.eu/property/periodName": {\
"class_field": "PeriodName",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/until": {\
"https://www.ariadne-infrastructure.eu/property/until": {\
"class_field": "Until",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/uri": {\
"https://www.ariadne-infrastructure.eu/property/uri": {\
"class_field": "Uri",\
"substring": "no",\
"element_type": "java.lang.String"\
@ -155,12 +46,12 @@ class.map.specifications={\
"AriadneGeoPoint": {\
"class_type": "unique",\
"mappings": {\
"http://www.myprefix/lat": {\
"https://www.ariadne-infrastructure.eu/property/lat": {\
"class_field": "Lat",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/lon": {\
"https://www.ariadne-infrastructure.eu/property/lon": {\
"class_field": "Lon",\
"substring": "no",\
"element_type": "java.lang.String"\
@ -169,27 +60,37 @@ class.map.specifications={\
},\
"Spatial": {\
"class_type": "prototype",\
"mappings":{"http://www.myprefix/address": {\
"mappings":{"https://www.ariadne-infrastructure.eu/property/address": {\
"class_field": "Address",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/coordinateSystem": {\
"https://www.ariadne-infrastructure.eu/property/coordinateSystem": {\
"class_field": "CoordinateSystem",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/country": {\
"https://www.ariadne-infrastructure.eu/property/country": {\
"class_field": "Country",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/location": {\
"https://www.ariadne-infrastructure.eu/property/location": {\
"class_field": "Location",\
"external_reference": "AriadneGeoPoint",\
"substring": "no"\
},\
"http://www.myprefix/placeName": {\
"https://www.ariadne-infrastructure.eu/property/lat": {\
"class_field": "Lat",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/lon": {\
"class_field": "Lon",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/placeName": {\
"class_field": "PlaceName",\
"substring": "no",\
"element_type": "java.lang.String"\
@ -198,12 +99,12 @@ class.map.specifications={\
"ArcheologicalResourceType": {\
"class_type": "unique",\
"mappings": {\
"http://www.myprefix/id": {\
"https://www.ariadne-infrastructure.eu/property/id": {\
"class_field": "Id",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/name": {\
"https://www.ariadne-infrastructure.eu/property/name": {\
"class_field": "Name",\
"substring": "no",\
"element_type": "java.lang.String"\
@ -213,22 +114,22 @@ class.map.specifications={\
"AgentInfo": {\
"class_type": "prototype",\
"mappings": {\
"http://www.myprefix/name": {\
"https://www.ariadne-infrastructure.eu/property/name": {\
"class_field": "Name",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/type": {\
"https://www.ariadne-infrastructure.eu/property/type": {\
"class_field": "Type",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/email": {\
"https://www.ariadne-infrastructure.eu/property/email": {\
"class_field": "Email",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/phone": {\
"https://www.ariadne-infrastructure.eu/property/phone": {\
"class_field": "Phone",\
"substring": "no",\
"element_type": "java.lang.String"\
@ -238,108 +139,178 @@ class.map.specifications={\
"NativeSubject": {\
"class_type": "prototype",\
"mappings": {\
"http://www.myprefix/prefLabel": {\
"https://www.ariadne-infrastructure.eu/property/prefLabel": {\
"class_field": "PrefLabel",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/rdfAbout": {\
"https://www.ariadne-infrastructure.eu/property/rdfAbout": {\
"class_field": "RdfAbout",\
"substring": "yes",\
"element_type": "java.lang.String"\
}\
}\
},\
"DerivedSubject": {\
"class_type": "prototype",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/prefLabel": {\
"class_field": "PrefLabel",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/source": {\
"class_field": "Source",\
"substring": "yes",\
"element_type": "java.lang.String"\
}\
}\
},\
"AatSubject": {\
"class_type": "prototype",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/id": {\
"class_field": "Id",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/label": {\
"class_field": "Label",\
"substring": "yes",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/lang": {\
"class_field": "Lang",\
"substring": "yes",\
"element_type": "java.lang.String"\
}\
}\
},\
"AriadneCatalogEntry": {\
"class_type": "unique",\
"mappings": {\
"http://www.myprefix/accessPolicy": {\
"https://www.ariadne-infrastructure.eu/property/accessPolicy": {\
"class_field": "AccessPolicy",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/identifier": {\
"https://www.ariadne-infrastructure.eu/property/identifier": {\
"class_field": "Identifier",\
"substring": "yes",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/publisher": {\
"https://www.ariadne-infrastructure.eu/property/publisher": {\
"class_field": "Publisher",\
"external_reference": "AgentInfo",\
"substring": "no"\
},\
"http://www.myprefix/archeologicalResourceType": {\
"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {\
"class_field": "ArcheologicalResourceType",\
"external_reference": "ArcheologicalResourceType",\
"substring": "no"\
},\
"http://www.myprefix/issued": {\
"https://www.ariadne-infrastructure.eu/property/issued": {\
"class_field": "Issued",\
"element_type": "java.lang.String",\
"substring": "no"\
},\
"http://www.myprefix/resourceType": {\
"https://www.ariadne-infrastructure.eu/property/resourceType": {\
"class_field": "ResourceType",\
"element_type": "java.lang.String",\
"substring": "no"\
},\
"http://www.myprefix/modified": {\
"https://www.ariadne-infrastructure.eu/property/modified": {\
"class_field": "Modified",\
"element_type": "java.lang.String",\
"substring": "no"\
},\
"http://www.myprefix/nativeSubject": {\
"https://www.ariadne-infrastructure.eu/property/nativeSubject": {\
"class_field": "NativeSubject",\
"substring": "no",\
"external_reference": "NativeSubject"\
},\
"http://www.myprefix/spatial": {\
"https://www.ariadne-infrastructure.eu/property/derivedSubject": {\
"class_field": "DerivedSubject",\
"substring": "no",\
"external_reference": "DerivedSubject"\
},\
"https://www.ariadne-infrastructure.eu/property/aatSubjects": {\
"class_field": "AatSubjects",\
"substring": "no",\
"external_reference": "AatSubject"\
},\
"https://www.ariadne-infrastructure.eu/property/spatialRegion": {\
"class_field": "Spatial",\
"substring": "no",\
"external_reference": "Spatial"\
},\
"http://www.myprefix/landingPage": {\
"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": {\
"class_field": "Spatial",\
"substring": "no",\
"external_reference": "Spatial"\
},\
"https://www.ariadne-infrastructure.eu/property/landingPage": {\
"class_field": "LandingPage",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/title": {\
"https://www.ariadne-infrastructure.eu/property/placeName": {\
"class_field": "PlaceName",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/title": {\
"class_field": "Title",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/accessRights": {\
"https://www.ariadne-infrastructure.eu/property/accessRights": {\
"class_field": "AccessRights",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/description": {\
"https://www.ariadne-infrastructure.eu/property/description": {\
"class_field": "Description",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/contributor": {\
"https://www.ariadne-infrastructure.eu/property/creator": {\
"class_field": "Creator",\
"substring": "no",\
"external_reference": "AgentInfo"\
},\
"https://www.ariadne-infrastructure.eu/property/contributor": {\
"class_field": "Contributor",\
"substring": "no",\
"external_reference": "AgentInfo"\
},\
"http://www.myprefix/originalId": {\
"https://www.ariadne-infrastructure.eu/property/legalResponsible": {\
"class_field": "LegalResponsible",\
"substring": "no",\
"external_reference": "AgentInfo"\
},\
"https://www.ariadne-infrastructure.eu/property/owner": {\
"class_field": "Owner",\
"substring": "no",\
"external_reference": "AgentInfo"\
},\
"https://www.ariadne-infrastructure.eu/property/originalId": {\
"class_field": "OriginalId",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/temporal": {\
"https://www.ariadne-infrastructure.eu/property/uri": {\
"class_field": "Temporal",\
"substring": "no",\
"external_reference": "AriadneTemporal"\
},\
"http://www.myprefix/language": {\
"https://www.ariadne-infrastructure.eu/property/language": {\
"class_field": "Language",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"http://www.myprefix/partOf": {\
"class_field": "PartOf",\
"https://www.ariadne-infrastructure.eu/property/partOf": {\
"class_field": "IsPartOf",\
"substring": "no",\
"element_type": "java.lang.String"\
}\

55
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java Normal file
View File

@ -0,0 +1,55 @@
/**
*
*/
package eu.dnetlib.ariadneplus;
import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.core.io.ClassPathResource;
import java.util.Properties;
/**
* @author enrico.ottonello
*
*/
public class GraphDbReaderAndESIndexTest {
private RunSPARQLQueryService runSPQRLQuery;
@Test
@Ignore
public void readAndIndexTest() throws Exception {
final ClassPathResource resource = new ClassPathResource("application.properties");
Properties appProps = new Properties();
appProps.load(resource.getInputStream());
runSPQRLQuery = new RunSPARQLQueryService();
runSPQRLQuery.setupConnection(
appProps.getProperty("graphdb.writer.user"),
appProps.getProperty("graphdb.writer.pwd"),
appProps.getProperty("repository.url"),
appProps.getProperty("graphdb.repository"));
ParseRDFJSON parseRDFJSON = new ParseRDFJSON();
parseRDFJSON.setCatalogEntryJsonPath(appProps.getProperty("catalog.entry.path"));
runSPQRLQuery.setParser(parseRDFJSON);
ResourceManager resourceManager = new ResourceManager();
resourceManager.setup(
appProps.getProperty("type.path"),
appProps.getProperty("general.classpath"),
appProps.getProperty("exclude.predicates"),
appProps.getProperty("class.map.specifications")
);
resourceManager.init();
runSPQRLQuery.setResourceManager(resourceManager);
BulkUpload bulkUpload = new BulkUpload();
bulkUpload.init();
runSPQRLQuery.setBulkUpload(bulkUpload);
runSPQRLQuery.executeQueryGraph();
}
}

40
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderTest.java
View File

@ -1,40 +0,0 @@
/**
*
*/
package eu.dnetlib.ariadneplus;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.springframework.beans.factory.annotation.Autowired;
import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
import eu.dnetlib.ariadneplus.reader.RunSPARQLQueryService;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
/**
* @author enrico.ottonello
*
*/
@RunWith(JUnit4.class)
public class GraphDbReaderTest {
@Autowired
private RunSPARQLQueryService runSPQRLQuery;
@Autowired
private ParseRDFJSON parser;
@Autowired
private ResourceManager resourceManager;
@Autowired
private BulkUpload bulkUpload;
@Test
public void readTest() throws Exception {
runSPQRLQuery.setParser(parser);
runSPQRLQuery.setResourceManager(resourceManager);
runSPQRLQuery.setBulkUpload(bulkUpload);
runSPQRLQuery.executeQueryGraph();
}
}

319
dnet-ariadneplus-graphdb-publisher/test/resources/application.properties Normal file
View File

@ -0,0 +1,319 @@
#server.contextPath=/ariadneplus-graphdb
#update due to springboot 2.1.3
server.servlet.context-path=/ariadneplus-graphdb
server.port=8281
graphdb.serverUrl=http://localhost:7200/
graphdb.writer.user=writer
graphdb.writer.pwd=*****
graphdb.repository=ariadneplus-ts01
graphdb.sparqlUrl = http://localhost:7200/sparql
graphdb.baseURI=https://ariadne-infrastructure.eu/
repository.url=http://graphdb-test.ariadne.d4science.org:7200
catalog.entry.path=$[*][?(@['https://www.ariadne-infrastructure.eu/property/resourceType'][0]['value']=='provided record')]
general.classpath=eu.dnetlib.ariadneplus.elasticsearch.model.
type.path=https://www.ariadne-infrastructure.eu/property/resourceType
exclude.predicates=["https://www.ariadne-infrastructure.eu/property/resourceType", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"]
class.map.specifications={\
"AriadneTemporal": {\
"class_type": "prototype",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/from": {\
"class_field": "From",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/periodName": {\
"class_field": "PeriodName",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/until": {\
"class_field": "Until",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/uri": {\
"class_field": "Uri",\
"substring": "no",\
"element_type": "java.lang.String"\
}\
}\
},\
"AriadneGeoPoint": {\
"class_type": "unique",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/lat": {\
"class_field": "Lat",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/lon": {\
"class_field": "Lon",\
"substring": "no",\
"element_type": "java.lang.String"\
}\
}\
},\
"Spatial": {\
"class_type": "prototype",\
"mappings":{"https://www.ariadne-infrastructure.eu/property/address": {\
"class_field": "Address",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/coordinateSystem": {\
"class_field": "CoordinateSystem",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/country": {\
"class_field": "Country",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/location": {\
"class_field": "Location",\
"external_reference": "AriadneGeoPoint",\
"substring": "no"\
},\
"https://www.ariadne-infrastructure.eu/property/lat": {\
"class_field": "Lat",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/lon": {\
"class_field": "Lon",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/placeName": {\
"class_field": "PlaceName",\
"substring": "no",\
"element_type": "java.lang.String"\
}}\
},\
"ArcheologicalResourceType": {\
"class_type": "unique",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/id": {\
"class_field": "Id",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/name": {\
"class_field": "Name",\
"substring": "no",\
"element_type": "java.lang.String"\
}\
}\
},\
"AgentInfo": {\
"class_type": "prototype",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/name": {\
"class_field": "Name",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/type": {\
"class_field": "Type",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/email": {\
"class_field": "Email",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/phone": {\
"class_field": "Phone",\
"substring": "no",\
"element_type": "java.lang.String"\
}\
}\
},\
"NativeSubject": {\
"class_type": "prototype",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/prefLabel": {\
"class_field": "PrefLabel",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/rdfAbout": {\
"class_field": "RdfAbout",\
"substring": "yes",\
"element_type": "java.lang.String"\
}\
}\
},\
"DerivedSubject": {\
"class_type": "prototype",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/prefLabel": {\
"class_field": "PrefLabel",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/source": {\
"class_field": "Source",\
"substring": "yes",\
"element_type": "java.lang.String"\
}\
}\
},\
"AatSubject": {\
"class_type": "prototype",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/id": {\
"class_field": "Id",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/label": {\
"class_field": "Label",\
"substring": "yes",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/lang": {\
"class_field": "Lang",\
"substring": "yes",\
"element_type": "java.lang.String"\
}\
}\
},\
"AriadneCatalogEntry": {\
"class_type": "unique",\
"mappings": {\
"https://www.ariadne-infrastructure.eu/property/accessPolicy": {\
"class_field": "AccessPolicy",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/identifier": {\
"class_field": "Identifier",\
"substring": "yes",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/publisher": {\
"class_field": "Publisher",\
"external_reference": "AgentInfo",\
"substring": "no"\
},\
"https://www.ariadne-infrastructure.eu/property/archeologicalResourceType": {\
"class_field": "ArcheologicalResourceType",\
"external_reference": "ArcheologicalResourceType",\
"substring": "no"\
},\
"https://www.ariadne-infrastructure.eu/property/issued": {\
"class_field": "Issued",\
"element_type": "java.lang.String",\
"substring": "no"\
},\
"https://www.ariadne-infrastructure.eu/property/resourceType": {\
"class_field": "ResourceType",\
"element_type": "java.lang.String",\
"substring": "no"\
},\
"https://www.ariadne-infrastructure.eu/property/modified": {\
"class_field": "Modified",\
"element_type": "java.lang.String",\
"substring": "no"\
},\
"https://www.ariadne-infrastructure.eu/property/nativeSubject": {\
"class_field": "NativeSubject",\
"substring": "no",\
"external_reference": "NativeSubject"\
},\
"https://www.ariadne-infrastructure.eu/property/derivedSubject": {\
"class_field": "DerivedSubject",\
"substring": "no",\
"external_reference": "DerivedSubject"\
},\
"https://www.ariadne-infrastructure.eu/property/aatSubjects": {\
"class_field": "AatSubjects",\
"substring": "no",\
"external_reference": "AatSubject"\
},\
"https://www.ariadne-infrastructure.eu/property/spatialRegion": {\
"class_field": "Spatial",\
"substring": "no",\
"external_reference": "Spatial"\
},\
"https://www.ariadne-infrastructure.eu/property/spatialRegionPoint": {\
"class_field": "Spatial",\
"substring": "no",\
"external_reference": "Spatial"\
},\
"https://www.ariadne-infrastructure.eu/property/landingPage": {\
"class_field": "LandingPage",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/placeName": {\
"class_field": "PlaceName",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/title": {\
"class_field": "Title",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/accessRights": {\
"class_field": "AccessRights",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/description": {\
"class_field": "Description",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/creator": {\
"class_field": "Creator",\
"substring": "no",\
"external_reference": "AgentInfo"\
},\
"https://www.ariadne-infrastructure.eu/property/contributor": {\
"class_field": "Contributor",\
"substring": "no",\
"external_reference": "AgentInfo"\
},\
"https://www.ariadne-infrastructure.eu/property/legalResponsible": {\
"class_field": "LegalResponsible",\
"substring": "no",\
"external_reference": "AgentInfo"\
},\
"https://www.ariadne-infrastructure.eu/property/owner": {\
"class_field": "Owner",\
"substring": "no",\
"external_reference": "AgentInfo"\
},\
"https://www.ariadne-infrastructure.eu/property/originalId": {\
"class_field": "OriginalId",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/uri": {\
"class_field": "Temporal",\
"substring": "no",\
"external_reference": "AriadneTemporal"\
},\
"https://www.ariadne-infrastructure.eu/property/language": {\
"class_field": "Language",\
"substring": "no",\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/partOf": {\
"class_field": "IsPartOf",\
"substring": "no",\
"element_type": "java.lang.String"\
}\
}\
}\
}