start aligning es json fields to es mapping
This commit is contained in:
parent
74d0f440e3
commit
9737184b1e
|
@ -2,6 +2,7 @@ package eu.dnetlib.ariadneplus.elasticsearch;
|
||||||
|
|
||||||
import eu.dnetlib.ariadneplus.elasticsearch.model.AgentInfo;
|
import eu.dnetlib.ariadneplus.elasticsearch.model.AgentInfo;
|
||||||
import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry;
|
import eu.dnetlib.ariadneplus.elasticsearch.model.AriadneCatalogEntry;
|
||||||
|
import eu.dnetlib.ariadneplus.elasticsearch.model.Spatial;
|
||||||
import eu.dnetlib.ariadneplus.reader.ResourceManager;
|
import eu.dnetlib.ariadneplus.reader.ResourceManager;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
@ -19,6 +20,7 @@ import org.springframework.stereotype.Service;
|
||||||
import javax.annotation.PostConstruct;
|
import javax.annotation.PostConstruct;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.lang.reflect.InvocationTargetException;
|
import java.lang.reflect.InvocationTargetException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class BulkUpload {
|
public class BulkUpload {
|
||||||
|
@ -49,17 +51,37 @@ public class BulkUpload {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void index(ResourceManager manager) {
|
public void index(ResourceManager manager, boolean isCollection) {
|
||||||
BulkRequest request = new BulkRequest();
|
BulkRequest request = new BulkRequest();
|
||||||
while (manager.hasNext()){
|
while (manager.hasNext()){
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Object next = manager.next();
|
Object next = manager.next();
|
||||||
AriadneCatalogEntry ace = ((AriadneCatalogEntry) next);
|
AriadneCatalogEntry ace = ((AriadneCatalogEntry) next);
|
||||||
|
if (isCollection) {
|
||||||
|
ace.setResourceType("collection");
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
ace.setResourceType("dataset");
|
||||||
|
}
|
||||||
|
Spatial esSpatial = new Spatial();
|
||||||
|
ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).forEach(s -> {
|
||||||
|
esSpatial.setPlaceName(s.getPlaceName());
|
||||||
|
});
|
||||||
|
ace.getSpatial().stream().filter(s -> s.getLocation()!=null).forEach(s -> {
|
||||||
|
esSpatial.setLocation(s.getLocation());
|
||||||
|
});
|
||||||
|
ace.getSpatial().clear();
|
||||||
|
ace.setSpatial(Arrays.asList(esSpatial));
|
||||||
|
String uniqueIsPartOf = ace.getUniqueIsPartOf();
|
||||||
|
if (uniqueIsPartOf!=null) {
|
||||||
|
ace.setIsPartOf(Arrays.asList(uniqueIsPartOf));
|
||||||
|
}
|
||||||
AgentInfo testPublisher = new AgentInfo();
|
AgentInfo testPublisher = new AgentInfo();
|
||||||
testPublisher.setName("TEST");
|
testPublisher.setName("TEST");
|
||||||
ace.getPublisher().add(testPublisher);
|
ace.getPublisher().add(testPublisher);
|
||||||
String[] splits = ace.getIdentifier().split("/");
|
String[] splits = ace.getIdentifier().split("/");
|
||||||
|
log.debug("indexing: "+ace.toJson());
|
||||||
request.add(new IndexRequest(elasticSearchIndexName).id(splits[splits.length-1])
|
request.add(new IndexRequest(elasticSearchIndexName).id(splits[splits.length-1])
|
||||||
.source(ace.toJson(),XContentType.JSON));
|
.source(ace.toJson(),XContentType.JSON));
|
||||||
log.debug("Indexing to ES: "+ace.toJson());
|
log.debug("Indexing to ES: "+ace.toJson());
|
||||||
|
|
|
@ -9,7 +9,7 @@ public class AgentInfo {
|
||||||
private String email;
|
private String email;
|
||||||
private String name;
|
private String name;
|
||||||
private String phone;
|
private String phone;
|
||||||
private String type;
|
private String type = new String("");
|
||||||
|
|
||||||
public AgentInfo() {
|
public AgentInfo() {
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,8 @@ public class AriadneCatalogEntry {
|
||||||
private List<ItemMetadataStructure> hasItemMetadataStructure;
|
private List<ItemMetadataStructure> hasItemMetadataStructure;
|
||||||
private List<MetadataRecord> hasMetadataRecord;
|
private List<MetadataRecord> hasMetadataRecord;
|
||||||
private String identifier;
|
private String identifier;
|
||||||
private String isPartOf;
|
private List<String> isPartOf;
|
||||||
|
private transient String uniqueIsPartOf;
|
||||||
private String issued;
|
private String issued;
|
||||||
private List<String> keyword;
|
private List<String> keyword;
|
||||||
private String landingPage;
|
private String landingPage;
|
||||||
|
@ -32,10 +33,10 @@ public class AriadneCatalogEntry {
|
||||||
private List<NativeSubject> nativeSubject;
|
private List<NativeSubject> nativeSubject;
|
||||||
private String originalId;
|
private String originalId;
|
||||||
private List<AgentInfo> owner;
|
private List<AgentInfo> owner;
|
||||||
private String packageId;
|
private String packageId = new String("");;
|
||||||
private String placeName;
|
private String placeName;
|
||||||
private String postcode;
|
private String postcode;
|
||||||
private String providerId;
|
private String providerId = new String("");;
|
||||||
private String rdfAbout;
|
private String rdfAbout;
|
||||||
private String resourceType;
|
private String resourceType;
|
||||||
private String rights;
|
private String rights;
|
||||||
|
@ -176,11 +177,11 @@ public class AriadneCatalogEntry {
|
||||||
this.identifier = identifier;
|
this.identifier = identifier;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getIsPartOf() {
|
public List<String> getIsPartOf() {
|
||||||
return isPartOf;
|
return isPartOf;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setIsPartOf(String isPartOf) {
|
public void setIsPartOf(List<String> isPartOf) {
|
||||||
this.isPartOf = isPartOf;
|
this.isPartOf = isPartOf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -213,7 +214,12 @@ public class AriadneCatalogEntry {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setLanguage(String language) {
|
public void setLanguage(String language) {
|
||||||
this.language = language;
|
if (language!=null && language.equals("eng")) {
|
||||||
|
this.language = "en";
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
this.language = language;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<AgentInfo> getLegalResponsible() {
|
public List<AgentInfo> getLegalResponsible() {
|
||||||
|
@ -349,7 +355,13 @@ public class AriadneCatalogEntry {
|
||||||
this.temporal = temporal;
|
this.temporal = temporal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getUniqueIsPartOf() {
|
||||||
|
return uniqueIsPartOf;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUniqueIsPartOf(String uniqueIsPartOf) {
|
||||||
|
this.uniqueIsPartOf = uniqueIsPartOf;
|
||||||
|
}
|
||||||
|
|
||||||
public static AriadneCatalogEntry fromJson(String json){
|
public static AriadneCatalogEntry fromJson(String json){
|
||||||
return new Gson().fromJson(json, AriadneCatalogEntry.class);
|
return new Gson().fromJson(json, AriadneCatalogEntry.class);
|
||||||
|
|
|
@ -6,22 +6,22 @@ import com.google.gson.JsonElement;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
public class AriadneGeoPoint {
|
public class AriadneGeoPoint {
|
||||||
private String lat;
|
private float lat;
|
||||||
private String lon;
|
private float lon;
|
||||||
|
|
||||||
public String getLat() {
|
public float getLat() {
|
||||||
return lat;
|
return lat;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setLat(String lat) {
|
public void setLat(float lat) {
|
||||||
this.lat = lat;
|
this.lat = lat;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getLon() {
|
public float getLon() {
|
||||||
return lon;
|
return lon;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setLon(String lon) {
|
public void setLon(float lon) {
|
||||||
this.lon = lon;
|
this.lon = lon;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -32,18 +32,18 @@ public class AriadneGeoPoint {
|
||||||
return new Gson().fromJson(json, AriadneGeoPoint.class);
|
return new Gson().fromJson(json, AriadneGeoPoint.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static AriadneGeoPoint fromRDFJson(JsonElement json){
|
// public static AriadneGeoPoint fromRDFJson(JsonElement json){
|
||||||
AriadneGeoPoint agp = new AriadneGeoPoint();
|
// AriadneGeoPoint agp = new AriadneGeoPoint();
|
||||||
for (Map.Entry<String, JsonElement> stringJsonElementEntry : json.getAsJsonObject().entrySet()) {
|
// for (Map.Entry<String, JsonElement> stringJsonElementEntry : json.getAsJsonObject().entrySet()) {
|
||||||
switch (stringJsonElementEntry.getKey()){
|
// switch (stringJsonElementEntry.getKey()){
|
||||||
case "https://www.ariadne-infrastructure.eu/property/lat":
|
// case "https://www.ariadne-infrastructure.eu/property/lat":
|
||||||
agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
|
// agp.setLat(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
|
||||||
break;
|
// break;
|
||||||
case "https://www.ariadne-infrastructure.eu/property/lon":
|
// case "https://www.ariadne-infrastructure.eu/property/lon":
|
||||||
agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
|
// agp.setLon(stringJsonElementEntry.getValue().getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString());
|
||||||
break;
|
// break;
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
return agp;
|
// return agp;
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,8 +12,8 @@ public class Spatial {
|
||||||
private String boundingBoxMaxLon;
|
private String boundingBoxMaxLon;
|
||||||
private String boundingBoxMinLat;
|
private String boundingBoxMinLat;
|
||||||
private String boundingBoxMinLon;
|
private String boundingBoxMinLon;
|
||||||
private String coordinateSystem;
|
private String coordinateSystem = new String("");;
|
||||||
private String country;
|
private String country = new String("");;
|
||||||
private AriadneGeoPoint location;
|
private AriadneGeoPoint location;
|
||||||
private String placeName;
|
private String placeName;
|
||||||
private String lat;
|
private String lat;
|
||||||
|
@ -95,14 +95,18 @@ public class Spatial {
|
||||||
if (this.getLocation()==null) {
|
if (this.getLocation()==null) {
|
||||||
this.setLocation(new AriadneGeoPoint());
|
this.setLocation(new AriadneGeoPoint());
|
||||||
}
|
}
|
||||||
this.getLocation().setLat(lat);
|
if (lat!=null) {
|
||||||
|
this.getLocation().setLat(Float.parseFloat(lat));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setLon(String lon) {
|
public void setLon(String lon) {
|
||||||
if (this.getLocation()==null) {
|
if (this.getLocation()==null) {
|
||||||
this.setLocation(new AriadneGeoPoint());
|
this.setLocation(new AriadneGeoPoint());
|
||||||
}
|
}
|
||||||
this.getLocation().setLon(lon);
|
if (lon!=null) {
|
||||||
|
this.getLocation().setLon(Float.parseFloat(lon));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public Spatial() {
|
public Spatial() {
|
||||||
|
|
|
@ -71,8 +71,9 @@ public class RunSPARQLQueryService {
|
||||||
|
|
||||||
private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){
|
private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){
|
||||||
log.debug("Retrieving "+recordId+" - isCollection:"+isCollection );
|
log.debug("Retrieving "+recordId+" - isCollection:"+isCollection );
|
||||||
String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">");
|
String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">");
|
||||||
openConnection();
|
log.debug(query);
|
||||||
|
openConnection();
|
||||||
StringWriter recordWriter = null;
|
StringWriter recordWriter = null;
|
||||||
Model resultsModel = null;
|
Model resultsModel = null;
|
||||||
String jsonRecord = null;
|
String jsonRecord = null;
|
||||||
|
@ -95,7 +96,7 @@ public class RunSPARQLQueryService {
|
||||||
}
|
}
|
||||||
parser.parse(recordWriter.toString());
|
parser.parse(recordWriter.toString());
|
||||||
resourceManager.manage(parser);
|
resourceManager.manage(parser);
|
||||||
bulkUpload.index(resourceManager);
|
bulkUpload.index(resourceManager, isCollection);
|
||||||
} catch(Exception e){
|
} catch(Exception e){
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
} finally{
|
} finally{
|
||||||
|
|
|
@ -310,7 +310,7 @@ class.map.specifications={\
|
||||||
"element_type": "java.lang.String"\
|
"element_type": "java.lang.String"\
|
||||||
},\
|
},\
|
||||||
"https://www.ariadne-infrastructure.eu/property/partOf": {\
|
"https://www.ariadne-infrastructure.eu/property/partOf": {\
|
||||||
"class_field": "IsPartOf",\
|
"class_field": "UniqueIsPartOf",\
|
||||||
"substring": "no",\
|
"substring": "no",\
|
||||||
"element_type": "java.lang.String"\
|
"element_type": "java.lang.String"\
|
||||||
}\
|
}\
|
||||||
|
|
|
@ -52,13 +52,13 @@ public class GraphDbReaderAndESIndexTest {
|
||||||
BulkUpload bulkUpload = new BulkUpload();
|
BulkUpload bulkUpload = new BulkUpload();
|
||||||
bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname"));
|
bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname"));
|
||||||
runSPQRLQuery.setBulkUpload(bulkUpload);
|
runSPQRLQuery.setBulkUpload(bulkUpload);
|
||||||
String recordId = "<https://ariadne-infrastructure.eu/aocat/Resource/C6D951BA-069B-3E39-B93C-9BAE2C48B280>";
|
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0320AA03-BED9-3F2B-AF5D-E31EE50CAE97";
|
||||||
String datasource = "ads";
|
String datasource = "ads";
|
||||||
String collectionId = "271";
|
String collectionId = "271";
|
||||||
List<String> recordIds = Arrays.asList(recordId);
|
List<String> recordIds = Arrays.asList(recordId);
|
||||||
final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
|
final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
|
||||||
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
|
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
|
||||||
runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId);
|
runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue