D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merging placename data with geopoint data and duplicate removing also for collection (MIBAC case)

This commit is contained in:
Enrico Ottonello 2022-06-21 11:58:56 +02:00
parent b7ed14928d
commit 0bee268468
2 changed files with 40 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java
View File

@ -150,26 +150,28 @@ public class BulkUpload {
.forEach(s -> {
s.setPolygon(s.getWkt());
});
if (ace.getSpatial().size()==2) {
Spatial uniqueSpatial = new Spatial();
boolean uniquePlaceNameFound = ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).count()==1;
boolean uniqueLocationFound = ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).count()==1;
if (uniquePlaceNameFound&&uniqueLocationFound) {
ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).forEach(s -> {
uniqueSpatial.setPlaceName(s.getPlaceName());
});
ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).forEach(s -> {
uniqueSpatial.setGeopoint(s.getGeopoint());
});
ace.getSpatial().clear();
ace.setSpatial(Arrays.asList(uniqueSpatial));
}
}
List<Spatial> dedupSpatials = removeDuplicates(ace.getSpatial());
ace.getSpatial().clear();
ace.setSpatial(dedupSpatials);
}
}
if (ace.getSpatial()!=null) {
if (ace.getSpatial().size()==2) {
Spatial uniqueSpatial = new Spatial();
boolean uniquePlaceNameFound = ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).count()==1;
boolean uniqueLocationFound = ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).count()==1;
if (uniquePlaceNameFound&&uniqueLocationFound) {
ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).forEach(s -> {
uniqueSpatial.setPlaceName(s.getPlaceName());
});
ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).forEach(s -> {
uniqueSpatial.setGeopoint(s.getGeopoint());
});
ace.getSpatial().clear();
ace.setSpatial(Arrays.asList(uniqueSpatial));
}
}
List<Spatial> dedupSpatials = removeDuplicates(ace.getSpatial());
ace.getSpatial().clear();
ace.setSpatial(dedupSpatials);
}
String[] splits = ace.getIdentifier().split("/");

24
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java
View File

@ -358,11 +358,9 @@ public class GraphDbReaderAndESIndexTest {
@Test
public void uploadDoliaTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/F0E19AC1-AE02-3BE5-AF78-D9EEC1670AC1";
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/EEAA72F8-3C33-3211-A56B-296164B5A317";
String datasource = "dolia";
String collectionId = "fieldworkreport";
readAndIndexTest(isRecord, recordId, datasource, collectionId);
recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/28630596-6EFF-38D7-91C7-8A6134CBD91B";
String collectionId = "full";
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
@ -402,6 +400,24 @@ public class GraphDbReaderAndESIndexTest {
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
@Test
public void uploadMibacTest() throws Exception {
boolean isRecord = false;
String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/MiC/662BD48F-7313-3017-8040-14CFFE1A02EB";
String datasource = "mibac";
String collectionId = "test";
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
@Test
public void uploadBMPASTest() throws Exception {
boolean isRecord = true;
String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/BMPAS/A561885D-2480-3794-82AD-A06B7462DA74";
String datasource = "bmpas";
String collectionId = "coins";
readAndIndexTest(isRecord, recordId, datasource, collectionId);
}
private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception {
final ClassPathResource resource = new ClassPathResource("application.properties");
Properties appProps = new Properties();