From f88231ba182c1233e3204016f721619aa33ec427 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Fri, 16 Oct 2020 19:37:21 +0200 Subject: [PATCH] added new methods to retrieve resource identifiers and to start indexing by resource identifiers --- .../ariadneplus/graphdb/GraphDBClient.java | 54 +++++++++++++++++-- .../AriadnePlusPublisherController.java | 16 ++++++ .../publisher/AriadnePlusPublisherHelper.java | 12 +++++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java index 533c0d0..7ae2d9f 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java @@ -1,12 +1,11 @@ package eu.dnetlib.ariadneplus.graphdb; import java.io.File; +import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; +import java.util.*; import com.google.common.base.Splitter; import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; @@ -375,4 +374,53 @@ public class GraphDBClient { } return "Records: ".concat(recordsIndexReport).concat(" Collection: ").concat(collectionIndexReport); } + + public List selectIdentifiers(String datasource, String collectionId, String resourceType) throws AriadnePlusPublisherException { + List identifiers = Collections.emptyList(); + try { + log.info("Select "+resourceType+" Identifiers from "+ datasource + " " + collectionId); + if (resourceType.equals("COLLECTION")) { + identifiers = runSPQRLQuery.selectCollectionIds(datasource, collectionId); + log.info(String.format("Found %d collections for datasource %s - %s", identifiers.size(), datasource, collectionId)); + } + else if (resourceType.equals("RECORD")) { + identifiers = runSPQRLQuery.selectRecordIds(datasource, collectionId); + log.info(String.format("Found %d records for datasource %s - %s", identifiers.size(), datasource, collectionId)); + } + }catch(Throwable e){ + log.error(e); + throw new AriadnePlusPublisherException(e); + } + return identifiers; + } + + public String indexOnESByIdentifiers(String datasource, String collectionId, String resourceType, List identifiers) throws AriadnePlusPublisherException { + String report = ""; + try { + log.info("Start indexing from "+ datasource + " " + collectionId); + runSPQRLQuery.setupConnection( getWriterUser(), getWriterPwd(), this.graphDBServerUrl, getRepository()); + runSPQRLQuery.setParser(parseRDFJSON); + runSPQRLQuery.setResourceManager(resourceManager); + runSPQRLQuery.setBulkUpload(bulkUpload); + + if (resourceType.equals("COLLECTION")) { + if (!identifiers.isEmpty()) { + final ClassPathResource selectCollectionTemplateRes = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql"); + String selectCollectionTemplate = IOUtils.toString(selectCollectionTemplateRes.getInputStream(), StandardCharsets.UTF_8.name()); + report = runSPQRLQuery.executeMultipleQueryGraph(selectCollectionTemplate, identifiers, datasource, collectionId, true); + } + } + else if (resourceType.equals("RECORD")) { + if(!identifiers.isEmpty()) { + final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); + String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); + report = runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, identifiers, datasource, collectionId, false); + } + } + } + catch (RuntimeException | IOException re) { + throw new AriadnePlusPublisherException(re); + } + return "Resources: ".concat(report); + } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java index 08f9168..98f2ad5 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java @@ -7,6 +7,8 @@ import org.springframework.web.bind.annotation.*; import eu.dnetlib.ariadneplus.publisher.AriadnePlusPublisherHelper.AriadnePlusTargets; +import java.util.List; + /** * @author enrico.ottonello * @@ -73,4 +75,18 @@ public class AriadnePlusPublisherController { public String indexOnES(@RequestParam final String datasource, @RequestParam final String collectionId) throws AriadnePlusPublisherException { return getAriadnePlusPublisherHelper().indexOnES(datasource, collectionId, getTarget(DEFAULT_TARGET_ENDPOINT)); } + + @RequestMapping(value = "/selectIdentifiers", method = RequestMethod.GET) + public List selectIdentifiers(@RequestParam final String datasource, @RequestParam final String collectionId, + @RequestParam String resourceType) throws AriadnePlusPublisherException { + return getAriadnePlusPublisherHelper().selectIdentifiers(datasource, collectionId, + resourceType, getTarget(DEFAULT_TARGET_ENDPOINT)); + } + + @RequestMapping(value = "/indexOnESByIdentifiers", method = RequestMethod.POST) + public String indexOnESByIdentifiers(@RequestParam final String datasource, @RequestParam final String collectionId, + @RequestParam String resourceType, @RequestParam List identifiers) throws AriadnePlusPublisherException { + return getAriadnePlusPublisherHelper().indexOnESByIdentifiers(datasource, collectionId, + resourceType, identifiers, getTarget(DEFAULT_TARGET_ENDPOINT)); + } } \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java index 93ecca3..6f8e526 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java @@ -8,6 +8,8 @@ import org.springframework.stereotype.Component; import eu.dnetlib.ariadneplus.graphdb.GraphDBClient; import eu.dnetlib.ariadneplus.graphdb.GraphDBClientFactory; +import java.util.List; + /** * @author enrico.ottonello * @@ -139,4 +141,14 @@ public class AriadnePlusPublisherHelper { GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); return graphDBClient.indexOnES(datasource, collectionId); } + + public List selectIdentifiers(String datasource, String collectionId, String resourceType, AriadnePlusTargets target) throws AriadnePlusPublisherException { + GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); + return graphDBClient.selectIdentifiers(datasource, collectionId, resourceType); + } + + public String indexOnESByIdentifiers(String datasource, String collectionId, String resourceType, List identifiers, AriadnePlusTargets target) throws AriadnePlusPublisherException { + GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); + return graphDBClient.indexOnESByIdentifiers(datasource, collectionId, resourceType, identifiers); + } }