D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

removed empty values/structures from record/collection to es

This commit is contained in:
Enrico Ottonello 2020-07-17 17:42:08 +02:00
parent d6aafdf97d
commit 766f8199f2
4 changed files with 49 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java
View File

@ -84,31 +84,29 @@ public class BulkUpload {
ace.getContributor().clear(); ace.getContributor().clear();
ace.setContributor(ace.getCreator()); ace.setContributor(ace.getCreator());
} }
Distribution distribution = new Distribution(); // Distribution distribution = new Distribution();
AgentInfo distrPublisher = new AgentInfo(); // AgentInfo distrPublisher = new AgentInfo();
distrPublisher.setEmail(""); // distrPublisher.setEmail("");
distrPublisher.setName(""); // distrPublisher.setName("");
distrPublisher.setType(""); // distrPublisher.setType("");
distribution.setPublisher(Arrays.asList(distrPublisher)); // distribution.setPublisher(Arrays.asList(distrPublisher));
ace.setDistribution(Arrays.asList(distribution)); // ace.setDistribution(Arrays.asList(distribution));
ItemMetadataStructure ims = new ItemMetadataStructure(); // ItemMetadataStructure ims = new ItemMetadataStructure();
ace.setHasItemMetadataStructure(Arrays.asList(ims)); // ace.setHasItemMetadataStructure(Arrays.asList(ims));
MetadataRecord mr = new MetadataRecord(); // MetadataRecord mr = new MetadataRecord();
Dex dex = new Dex(); // Dex dex = new Dex();
mr.setConformsTo(Arrays.asList(dex)); // mr.setConformsTo(Arrays.asList(dex));
ace.setHasMetadataRecord(Arrays.asList(mr)); // ace.setHasMetadataRecord(Arrays.asList(mr));
if (!isCollection) { // if (!isCollection) {
ace.setKeyword(Arrays.asList(new String(""))); // ace.setKeyword(Arrays.asList(new String("")));
} // }
AgentInfo sr = new AgentInfo(); // AgentInfo sr = new AgentInfo();
ace.setScientificResponsible(Arrays.asList(sr)); // ace.setScientificResponsible(Arrays.asList(sr));
AgentInfo tr = new AgentInfo(); // AgentInfo tr = new AgentInfo();
ace.setTechnicalResponsible(Arrays.asList(tr)); // ace.setTechnicalResponsible(Arrays.asList(tr));
} }
AgentInfo testPublisher = new AgentInfo(); AgentInfo testPublisher = new AgentInfo();
testPublisher.setName("TEST"); testPublisher.setName("TEST");
testPublisher.setPhone(null);
testPublisher.setEmail(null);
ace.getPublisher().add(testPublisher); ace.getPublisher().add(testPublisher);
String[] splits = ace.getIdentifier().split("/"); String[] splits = ace.getIdentifier().split("/");
@ -122,13 +120,7 @@ public class BulkUpload {
ace.getAatSubjects().forEach(s -> s.setId(aatSubjectId)); ace.getAatSubjects().forEach(s -> s.setId(aatSubjectId));
} }
String idES; String idES = splits[splits.length-1];
if (isCollection) {
idES = "10000".concat(ace.getOriginalId());
}
else {
idES = splits[splits.length-1];
}
request.add(new IndexRequest(elasticSearchIndexName).id(idES) request.add(new IndexRequest(elasticSearchIndexName).id(idES)
.source(ace.toJson(),XContentType.JSON)); .source(ace.toJson(),XContentType.JSON));

16
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java
View File

@ -39,16 +39,16 @@ public class AriadneCatalogEntry {
private List<AgentInfo> technicalResponsible; private List<AgentInfo> technicalResponsible;
private List<AriadneTemporal> temporal; private List<AriadneTemporal> temporal;
private String accrualPeriodicity = new String(""); private String accrualPeriodicity;// = new String("");
private String audience;// IF PRESENT, MUST CONTAIN A VALUE !! private String audience;// IF PRESENT, MUST CONTAIN A VALUE !!
private String contactPoint = new String(""); private String contactPoint;// = new String("");
private String extent;// IF PRESENT, MUST CONTAIN A VALUE !! private String extent;// IF PRESENT, MUST CONTAIN A VALUE !!
private String providerId = new String(""); private String providerId;// = new String("");
private String packageId = new String(""); private String packageId;// = new String("");
private String placeName = new String(""); private String placeName;// = new String("");
private String postcode = new String(""); private String postcode;// = new String("");
private String rdfAbout = new String(""); private String rdfAbout;// = new String("");
private String rights = new String(""); private String rights;// = new String("");
public List<AgentInfo> getContributor() { public List<AgentInfo> getContributor() {
return contributor; return contributor;

4
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/Spatial.java
View File

@ -7,13 +7,13 @@ import java.util.Map;
public class Spatial { public class Spatial {
private String address = new String(""); private String address;// = new String("");
private String boundingBoxMaxLat; private String boundingBoxMaxLat;
private String boundingBoxMaxLon; private String boundingBoxMaxLon;
private String boundingBoxMinLat; private String boundingBoxMinLat;
private String boundingBoxMinLon; private String boundingBoxMinLon;
private String coordinateSystem = new String(""); private String coordinateSystem = new String("");
private String country = new String(""); private String country;// = new String("");
private AriadneGeoPoint location; private AriadneGeoPoint location;
private String placeName; private String placeName;
private String lat; private String lat;

24
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java
View File

@ -53,16 +53,28 @@ public class GraphDbReaderAndESIndexTest {
BulkUpload bulkUpload = new BulkUpload(); BulkUpload bulkUpload = new BulkUpload();
bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname")); bulkUpload.init(appProps.getProperty("elasticsearch.hostname"),appProps.getProperty("elasticsearch.indexname"));
runSPQRLQuery.setBulkUpload(bulkUpload); runSPQRLQuery.setBulkUpload(bulkUpload);
// String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/849B1C0F-4C5F-3D8C-9082-CA60DBB4F557"; String recordId;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0320AA03-BED9-3F2B-AF5D-E31EE50CAE97"; final ClassPathResource queryTemplateResource;
boolean testRecord = false;
if (testRecord) {
recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0320AA03-BED9-3F2B-AF5D-E31EE50CAE97";
queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
}
else {
recordId = "https://ariadne-infrastructure.eu/aocat/Collection/ADS/849B1C0F-4C5F-3D8C-9082-CA60DBB4F557";
queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql");
}
String datasource = "ads"; String datasource = "ads";
String collectionId = "271"; String collectionId = "271";
List<String> recordIds = Arrays.asList(recordId); List<String> recordIds = Arrays.asList(recordId);
// final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql");
final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false); if (testRecord) {
// runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, true); runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false);
}
else {
runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, true);
}
} }
@Test @Test