D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

start aligning es json fields to es mapping

This commit is contained in:
Enrico Ottonello 2020-07-07 13:39:22 +02:00
parent 74d0f440e3
commit 9737184b1e
8 changed files with 77 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.ariadneplus.elasticsearch;
import eu.dnetlib.ariadneplus.elasticsearch.model.AgentInfo;
import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry;
import eu.dnetlib.ariadneplus.elasticsearch.model.Spatial;
import eu.dnetlib.ariadneplus.reader.ResourceManager;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -19,6 +20,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
@Service
public class BulkUpload {
@ -49,17 +51,37 @@ public class BulkUpload {
}
public void index(ResourceManager manager) {
public void index(ResourceManager manager, boolean isCollection) {
BulkRequest request = new BulkRequest();
while (manager.hasNext()){
try {
Object next = manager.next();
AriadneCatalogEntry ace = ((AriadneCatalogEntry) next);
if (isCollection) {
ace.setResourceType("collection");
}
else {
ace.setResourceType("dataset");
}
Spatial esSpatial = new Spatial();
ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).forEach(s -> {
esSpatial.setPlaceName(s.getPlaceName());
});
ace.getSpatial().stream().filter(s -> s.getLocation()!=null).forEach(s -> {
esSpatial.setLocation(s.getLocation());
});
ace.getSpatial().clear();
ace.setSpatial(Arrays.asList(esSpatial));
String uniqueIsPartOf = ace.getUniqueIsPartOf();
if (uniqueIsPartOf!=null) {
ace.setIsPartOf(Arrays.asList(uniqueIsPartOf));
}
AgentInfo testPublisher = new AgentInfo();
testPublisher.setName("TEST");
ace.getPublisher().add(testPublisher);
String[] splits = ace.getIdentifier().split("/");
log.debug("indexing: "+ace.toJson());
request.add(new IndexRequest(elasticSearchIndexName).id(splits[splits.length-1])
.source(ace.toJson(),XContentType.JSON));
log.debug("Indexing to ES: "+ace.toJson());

2
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AgentInfo.java
View File

@ -9,7 +9,7 @@ public class AgentInfo {
private String email;
private String name;
private String phone;
private String type;
private String type = new String("");
public AgentInfo() {

24
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java
View File

@ -22,7 +22,8 @@ public class AriadneCatalogEntry {
private List<ItemMetadataStructure> hasItemMetadataStructure;
private List<MetadataRecord> hasMetadataRecord;
private String identifier;
private String isPartOf;
private List<String> isPartOf;
private transient String uniqueIsPartOf;
private String issued;
private List<String> keyword;
private String landingPage;
@ -32,10 +33,10 @@ public class AriadneCatalogEntry {
private List<NativeSubject> nativeSubject;
private String originalId;
private List<AgentInfo> owner;
private String packageId;
private String packageId = new String("");;
private String placeName;
private String postcode;
private String providerId;
private String providerId = new String("");;
private String rdfAbout;
private String resourceType;
private String rights;
@ -176,11 +177,11 @@ public class AriadneCatalogEntry {
this.identifier = identifier;
}
public String getIsPartOf() {
public List<String> getIsPartOf() {
return isPartOf;
}
public void setIsPartOf(String isPartOf) {
public void setIsPartOf(List<String> isPartOf) {
this.isPartOf = isPartOf;
}
@ -213,7 +214,12 @@ public class AriadneCatalogEntry {
}
public void setLanguage(String language) {
this.language = language;
if (language!=null && language.equals("eng")) {
this.language = "en";
}
else {
this.language = language;
}
}
public List<AgentInfo> getLegalResponsible() {
@ -349,7 +355,13 @@ public class AriadneCatalogEntry {
this.temporal = temporal;
}
public String getUniqueIsPartOf() {
return uniqueIsPartOf;
}
public void setUniqueIsPartOf(String uniqueIsPartOf) {
this.uniqueIsPartOf = uniqueIsPartOf;
}
public static AriadneCatalogEntry fromJson(String json){
return new Gson().fromJson(json, AriadneCatalogEntry.class);

40
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneGeoPoint.java
View File

@ -6,22 +6,22 @@ import com.google.gson.JsonElement;
import java.util.Map;
public class AriadneGeoPoint {
private String lat;
private String lon;
private float lat;
private float lon;
public String getLat() {
public float getLat() {
return lat;
}
public void setLat(String lat) {
public void setLat(float lat) {
this.lat = lat;
}
public String getLon() {
public float getLon() {
return lon;
}
public void setLon(String lon) {
public void setLon(float lon) {
this.lon = lon;
}
@ -32,18 +32,18 @@ public class AriadneGeoPoint {
return new Gson().fromJson(json, AriadneGeoPoint.class);
}
public static AriadneGeoPoint fromRDFJson(JsonElement json){
AriadneGeoPoint agp = new AriadneGeoPoint();
for (Map.Entry<String, JsonElement> stringJsonElementEntry : json.getAsJsonObject().entrySet()) {
switch (stringJsonElementEntry.getKey()){
case "https://www.ariadne-infrastructure.eu/property/lat":
agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
case "https://www.ariadne-infrastructure.eu/property/lon":
agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
break;
}
}
return agp;
}
// public static AriadneGeoPoint fromRDFJson(JsonElement json){
// AriadneGeoPoint agp = new AriadneGeoPoint();
// for (Map.Entry<String, JsonElement> stringJsonElementEntry : json.getAsJsonObject().entrySet()) {
// switch (stringJsonElementEntry.getKey()){
// case "https://www.ariadne-infrastructure.eu/property/lat":
// agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// case "https://www.ariadne-infrastructure.eu/property/lon":
// agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
// break;
// }
// }
// return agp;
// }
}

12
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java
View File

@ -12,8 +12,8 @@ public class Spatial {
private String boundingBoxMaxLon;
private String boundingBoxMinLat;
private String boundingBoxMinLon;
private String coordinateSystem;
private String country;
private String coordinateSystem = new String("");;
private String country = new String("");;
private AriadneGeoPoint location;
private String placeName;
private String lat;
@ -95,14 +95,18 @@ public class Spatial {
if (this.getLocation()==null) {
this.setLocation(new AriadneGeoPoint());
}
this.getLocation().setLat(lat);
if (lat!=null) {
this.getLocation().setLat(Float.parseFloat(lat));
}
}
public void setLon(String lon) {
if (this.getLocation()==null) {
this.setLocation(new AriadneGeoPoint());
}
this.getLocation().setLon(lon);
if (lon!=null) {
this.getLocation().setLon(Float.parseFloat(lon));
}
}
public Spatial() {

7
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java
View File

@ -71,8 +71,9 @@ public class RunSPARQLQueryService {
private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){
log.debug("Retrieving "+recordId+" - isCollection:"+isCollection );
String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">");
openConnection();
String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">");
log.debug(query);
openConnection();
StringWriter recordWriter = null;
Model resultsModel = null;
String jsonRecord = null;
@ -95,7 +96,7 @@ public class RunSPARQLQueryService {
}
parser.parse(recordWriter.toString());
resourceManager.manage(parser);
bulkUpload.index(resourceManager);
bulkUpload.index(resourceManager, isCollection);
} catch(Exception e){
e.printStackTrace();
} finally{

2
dnet-ariadneplus-graphdb-publisher/src/main/resources/application.properties
View File

@ -310,7 +310,7 @@ class.map.specifications={\
"element_type": "java.lang.String"\
},\
"https://www.ariadne-infrastructure.eu/property/partOf": {\
"class_field": "IsPartOf",\
"class_field": "UniqueIsPartOf",\
"substring": "no",\
"element_type": "java.lang.String"\
}\

4
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java
View File

@ -52,13 +52,13 @@ public class GraphDbReaderAndESIndexTest {
BulkUpload bulkUpload = new BulkUpload();
bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname"));
runSPQRLQuery.setBulkUpload(bulkUpload);
String recordId = "<https://ariadne-infrastructure.eu/aocat/Resource/C6D951BA-069B-3E39-B93C-9BAE2C48B280>";
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0320AA03-BED9-3F2B-AF5D-E31EE50CAE97";
String datasource = "ads";
String collectionId = "271";
List<String> recordIds = Arrays.asList(recordId);
final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId);
runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false);
}