D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

added new methods to retrieve resource identifiers and to start indexing by resource identifiers

This commit is contained in:
Enrico Ottonello 2020-10-16 19:37:21 +02:00
parent 0e96774895
commit f88231ba18
3 changed files with 79 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java
View File

@ -1,12 +1,11 @@
package eu.dnetlib.ariadneplus.graphdb; package eu.dnetlib.ariadneplus.graphdb;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.Arrays; import java.util.*;
import java.util.List;
import java.util.Properties;
import com.google.common.base.Splitter; import com.google.common.base.Splitter;
import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload; import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
@ -375,4 +374,53 @@ public class GraphDBClient {
} }
return "Records: ".concat(recordsIndexReport).concat(" Collection: ").concat(collectionIndexReport); return "Records: ".concat(recordsIndexReport).concat(" Collection: ").concat(collectionIndexReport);
} }
public List<String> selectIdentifiers(String datasource, String collectionId, String resourceType) throws AriadnePlusPublisherException {
List <String> identifiers = Collections.emptyList();
try {
log.info("Select "+resourceType+" Identifiers from "+ datasource + " " + collectionId);
if (resourceType.equals("COLLECTION")) {
identifiers = runSPQRLQuery.selectCollectionIds(datasource, collectionId);
log.info(String.format("Found %d collections for datasource %s - %s", identifiers.size(), datasource, collectionId));
}
else if (resourceType.equals("RECORD")) {
identifiers = runSPQRLQuery.selectRecordIds(datasource, collectionId);
log.info(String.format("Found %d records for datasource %s - %s", identifiers.size(), datasource, collectionId));
}
}catch(Throwable e){
log.error(e);
throw new AriadnePlusPublisherException(e);
}
return identifiers;
}
public String indexOnESByIdentifiers(String datasource, String collectionId, String resourceType, List <String> identifiers) throws AriadnePlusPublisherException {
String report = "";
try {
log.info("Start indexing from "+ datasource + " " + collectionId);
runSPQRLQuery.setupConnection( getWriterUser(), getWriterPwd(), this.graphDBServerUrl, getRepository());
runSPQRLQuery.setParser(parseRDFJSON);
runSPQRLQuery.setResourceManager(resourceManager);
runSPQRLQuery.setBulkUpload(bulkUpload);
if (resourceType.equals("COLLECTION")) {
if (!identifiers.isEmpty()) {
final ClassPathResource selectCollectionTemplateRes = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_collection_data_template.sparql");
String selectCollectionTemplate = IOUtils.toString(selectCollectionTemplateRes.getInputStream(), StandardCharsets.UTF_8.name());
report = runSPQRLQuery.executeMultipleQueryGraph(selectCollectionTemplate, identifiers, datasource, collectionId, true);
}
}
else if (resourceType.equals("RECORD")) {
if(!identifiers.isEmpty()) {
final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
report = runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, identifiers, datasource, collectionId, false);
}
}
}
catch (RuntimeException | IOException re) {
throw new AriadnePlusPublisherException(re);
}
return "Resources: ".concat(report);
}
} }

16
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java
View File

@ -7,6 +7,8 @@ import org.springframework.web.bind.annotation.*;
import eu.dnetlib.ariadneplus.publisher.AriadnePlusPublisherHelper.AriadnePlusTargets; import eu.dnetlib.ariadneplus.publisher.AriadnePlusPublisherHelper.AriadnePlusTargets;
import java.util.List;
/** /**
* @author enrico.ottonello * @author enrico.ottonello
* *
@ -73,4 +75,18 @@ public class AriadnePlusPublisherController {
public String indexOnES(@RequestParam final String datasource, @RequestParam final String collectionId) throws AriadnePlusPublisherException { public String indexOnES(@RequestParam final String datasource, @RequestParam final String collectionId) throws AriadnePlusPublisherException {
return getAriadnePlusPublisherHelper().indexOnES(datasource, collectionId, getTarget(DEFAULT_TARGET_ENDPOINT)); return getAriadnePlusPublisherHelper().indexOnES(datasource, collectionId, getTarget(DEFAULT_TARGET_ENDPOINT));
} }
@RequestMapping(value = "/selectIdentifiers", method = RequestMethod.GET)
public List<String> selectIdentifiers(@RequestParam final String datasource, @RequestParam final String collectionId,
@RequestParam String resourceType) throws AriadnePlusPublisherException {
return getAriadnePlusPublisherHelper().selectIdentifiers(datasource, collectionId,
resourceType, getTarget(DEFAULT_TARGET_ENDPOINT));
}
@RequestMapping(value = "/indexOnESByIdentifiers", method = RequestMethod.POST)
public String indexOnESByIdentifiers(@RequestParam final String datasource, @RequestParam final String collectionId,
@RequestParam String resourceType, @RequestParam List<String> identifiers) throws AriadnePlusPublisherException {
return getAriadnePlusPublisherHelper().indexOnESByIdentifiers(datasource, collectionId,
resourceType, identifiers, getTarget(DEFAULT_TARGET_ENDPOINT));
}
} }

12
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java
View File

@ -8,6 +8,8 @@ import org.springframework.stereotype.Component;
import eu.dnetlib.ariadneplus.graphdb.GraphDBClient; import eu.dnetlib.ariadneplus.graphdb.GraphDBClient;
import eu.dnetlib.ariadneplus.graphdb.GraphDBClientFactory; import eu.dnetlib.ariadneplus.graphdb.GraphDBClientFactory;
import java.util.List;
/** /**
* @author enrico.ottonello * @author enrico.ottonello
* *
@ -139,4 +141,14 @@ public class AriadnePlusPublisherHelper {
GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient();
return graphDBClient.indexOnES(datasource, collectionId); return graphDBClient.indexOnES(datasource, collectionId);
} }
public List<String> selectIdentifiers(String datasource, String collectionId, String resourceType, AriadnePlusTargets target) throws AriadnePlusPublisherException {
GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient();
return graphDBClient.selectIdentifiers(datasource, collectionId, resourceType);
}
public String indexOnESByIdentifiers(String datasource, String collectionId, String resourceType, List<String> identifiers, AriadnePlusTargets target) throws AriadnePlusPublisherException {
GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient();
return graphDBClient.indexOnESByIdentifiers(datasource, collectionId, resourceType, identifiers);
}
} }