D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

added response code from es; added sparql insert for copying ariadne_subject value into collection for ads; fix sparql insert for copying collection data into records, collection id condition is needed

This commit is contained in:
Enrico Ottonello 2020-07-22 23:13:47 +02:00
parent 766f8199f2
commit 5bb65a8854
5 changed files with 40 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java
View File

@ -49,8 +49,9 @@ public class BulkUpload {
} }
public void index(ResourceManager manager, boolean isCollection) { public int index(ResourceManager manager, boolean isCollection) {
BulkRequest request = new BulkRequest(); BulkRequest request = new BulkRequest();
int esResponseCode = 0;
while (manager.hasNext()){ while (manager.hasNext()){
try { try {
@ -125,24 +126,18 @@ public class BulkUpload {
.source(ace.toJson(),XContentType.JSON)); .source(ace.toJson(),XContentType.JSON));
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
log.info("Indexing to ES completed with status: "+bulkResponse.status()); if (bulkResponse!=null) {
esResponseCode = bulkResponse.status().getStatus();
log.info("Indexing to ES completed with status: " + bulkResponse.status());
if (bulkResponse.hasFailures()) { if (bulkResponse.hasFailures()) {
log.error("FailureMessage: "+bulkResponse.buildFailureMessage()); log.error("FailureMessage: " + bulkResponse.buildFailureMessage());
}
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} }
} }
} catch (Exception e) {
log.error("Indexing "+e.getMessage());
return -1;
}
}
return esResponseCode;
} }
} }

18
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java
View File

@ -64,18 +64,18 @@ public class RunSPARQLQueryService {
return null; return null;
final String selectQueryTemplate = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); final String selectQueryTemplate = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId);
recordIds.forEach(recordId -> { recordIds.forEach(recordId -> {
executeQueryGraph(selectQueryTemplate, recordId, isCollection); int operationResult = executeQueryGraph(selectQueryTemplate, recordId, isCollection);
log.info("Indexing final code: "+Integer.toString(operationResult));
}); });
return "ok"; return "ok";
} }
private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){ private int executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){
log.debug("Retrieving "+recordId+" - isCollection:"+isCollection ); log.info("Retrieving "+recordId+" - isCollection:"+isCollection );
String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">"); String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">");
openConnection(); openConnection();
StringWriter recordWriter = null; StringWriter recordWriter = null;
Model resultsModel = null; Model resultsModel = null;
String jsonRecord = null;
try { try {
// log.debug("Started at: "+Calendar.getInstance().getTime().toString()); // log.debug("Started at: "+Calendar.getInstance().getTime().toString());
GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query); GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query);
@ -83,9 +83,9 @@ public class RunSPARQLQueryService {
resultsModel = QueryResults.asModel(graphQueryResult); resultsModel = QueryResults.asModel(graphQueryResult);
graphQueryResult.close(); graphQueryResult.close();
// log.debug("Finished at: "+Calendar.getInstance().getTime().toString()); // log.debug("Finished at: "+Calendar.getInstance().getTime().toString());
log.debug("Statements retrieved: " + resultsModel.size()); log.info("Statements retrieved: " + resultsModel.size());
if (resultsModel.size()==0) { if (resultsModel.size()==0) {
return "noresult"; return -2;
} }
recordWriter = new StringWriter(); recordWriter = new StringWriter();
RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter);
@ -95,16 +95,16 @@ public class RunSPARQLQueryService {
} }
parser.parse(recordWriter.toString()); parser.parse(recordWriter.toString());
resourceManager.manage(parser); resourceManager.manage(parser);
bulkUpload.index(resourceManager, isCollection); return bulkUpload.index(resourceManager, isCollection);
} catch(Exception e){ } catch(Exception e){
e.printStackTrace(); log.error(e);
return -1;
} finally{ } finally{
closeConnection(); closeConnection();
if (resultsModel!=null) { if (resultsModel!=null) {
resultsModel.clear(); resultsModel.clear();
} }
} }
return jsonRecord;
} }
public ParseRDFJSON getParser() { public ParseRDFJSON getParser() {

1
dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql
View File

@ -14,6 +14,7 @@ INSERT {
?record aocat:has_ARIADNE_subject ?archeologicalResourceType . ?record aocat:has_ARIADNE_subject ?archeologicalResourceType .
} }
} }
USING <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::271>
WHERE { WHERE {
?record aocat:is_part_of ?collection . ?record aocat:is_part_of ?collection .
?collection aocat:was_issued ?issued . ?collection aocat:was_issued ?issued .

15
dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/insert_ads_ariadne_subject.sparql Normal file
View File

@ -0,0 +1,15 @@
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::271> {
?collection aocat:has_ARIADNE_subject ?archeologicalResourceType .
?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .
}
}
WHERE {
?collection rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
?collection aocat:has_ARIADNE_subject ?archeologicalResourceType .
?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .
?collection aocat:has_original_id "1000271" .
}

6
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java
View File

@ -55,9 +55,9 @@ public class GraphDbReaderAndESIndexTest {
runSPQRLQuery.setBulkUpload(bulkUpload); runSPQRLQuery.setBulkUpload(bulkUpload);
String recordId; String recordId;
final ClassPathResource queryTemplateResource; final ClassPathResource queryTemplateResource;
boolean testRecord = false; boolean testRecord = true;
if (testRecord) { if (testRecord) {
recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0320AA03-BED9-3F2B-AF5D-E31EE50CAE97"; recordId = "https://ariadne-infrastructure.eu/aocat/Resource/10DB6647-5D40-397A-8434-8CED55B2FC29";
queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
} }
else { else {
@ -66,7 +66,7 @@ public class GraphDbReaderAndESIndexTest {
} }
String datasource = "ads"; String datasource = "ads";
String collectionId = "271"; String collectionId = "270";
List<String> recordIds = Arrays.asList(recordId); List<String> recordIds = Arrays.asList(recordId);
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
if (testRecord) { if (testRecord) {