From 0bee26846862b36350bb4e6b950ef2d55412a24d Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 21 Jun 2022 11:58:56 +0200 Subject: [PATCH] merging placename data with geopoint data and duplicate removing also for collection (MIBAC case) --- .../ariadneplus/elasticsearch/BulkUpload.java | 38 ++++++++++--------- .../GraphDbReaderAndESIndexTest.java | 24 ++++++++++-- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index be7d918..a69d2f6 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -150,26 +150,28 @@ public class BulkUpload { .forEach(s -> { s.setPolygon(s.getWkt()); }); - if (ace.getSpatial().size()==2) { - Spatial uniqueSpatial = new Spatial(); - boolean uniquePlaceNameFound = ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).count()==1; - boolean uniqueLocationFound = ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).count()==1; - if (uniquePlaceNameFound&&uniqueLocationFound) { - ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).forEach(s -> { - uniqueSpatial.setPlaceName(s.getPlaceName()); - }); - ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).forEach(s -> { - uniqueSpatial.setGeopoint(s.getGeopoint()); - }); - ace.getSpatial().clear(); - ace.setSpatial(Arrays.asList(uniqueSpatial)); - } - } - List dedupSpatials = removeDuplicates(ace.getSpatial()); - ace.getSpatial().clear(); - ace.setSpatial(dedupSpatials); } } + if (ace.getSpatial()!=null) { + if (ace.getSpatial().size()==2) { + Spatial uniqueSpatial = new Spatial(); + boolean uniquePlaceNameFound = ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).count()==1; + boolean uniqueLocationFound = ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).count()==1; + if (uniquePlaceNameFound&&uniqueLocationFound) { + ace.getSpatial().stream().filter(s -> s.getPlaceName()!=null).forEach(s -> { + uniqueSpatial.setPlaceName(s.getPlaceName()); + }); + ace.getSpatial().stream().filter(s -> s.getGeopoint()!=null).forEach(s -> { + uniqueSpatial.setGeopoint(s.getGeopoint()); + }); + ace.getSpatial().clear(); + ace.setSpatial(Arrays.asList(uniqueSpatial)); + } + } + List dedupSpatials = removeDuplicates(ace.getSpatial()); + ace.getSpatial().clear(); + ace.setSpatial(dedupSpatials); + } String[] splits = ace.getIdentifier().split("/"); diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 32023ad..36b321e 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -358,11 +358,9 @@ public class GraphDbReaderAndESIndexTest { @Test public void uploadDoliaTest() throws Exception { boolean isRecord = true; - String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/F0E19AC1-AE02-3BE5-AF78-D9EEC1670AC1"; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/EEAA72F8-3C33-3211-A56B-296164B5A317"; String datasource = "dolia"; - String collectionId = "fieldworkreport"; - readAndIndexTest(isRecord, recordId, datasource, collectionId); - recordId = "https://ariadne-infrastructure.eu/aocat/Resource/Inrap/28630596-6EFF-38D7-91C7-8A6134CBD91B"; + String collectionId = "full"; readAndIndexTest(isRecord, recordId, datasource, collectionId); } @@ -402,6 +400,24 @@ public class GraphDbReaderAndESIndexTest { readAndIndexTest(isRecord, recordId, datasource, collectionId); } + @Test + public void uploadMibacTest() throws Exception { + boolean isRecord = false; + String recordId = "https://ariadne-infrastructure.eu/aocat/Collection/MiC/662BD48F-7313-3017-8040-14CFFE1A02EB"; + String datasource = "mibac"; + String collectionId = "test"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + + @Test + public void uploadBMPASTest() throws Exception { + boolean isRecord = true; + String recordId = "https://ariadne-infrastructure.eu/aocat/Resource/BMPAS/A561885D-2480-3794-82AD-A06B7462DA74"; + String datasource = "bmpas"; + String collectionId = "coins"; + readAndIndexTest(isRecord, recordId, datasource, collectionId); + } + private void readAndIndexTest(boolean isRecord, String recordId, String datasource, String collectionId) throws Exception { final ClassPathResource resource = new ClassPathResource("application.properties"); Properties appProps = new Properties();