D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

added response code from es; added sparql insert for copying ariadne_subject value into collection for ads; fix sparql insert for copying collection data into records, collection id condition is needed

This commit is contained in:
Enrico Ottonello 2020-07-22 23:13:47 +02:00
parent 766f8199f2
commit 5bb65a8854
5 changed files with 40 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java
View File

@ -49,8 +49,9 @@ public class BulkUpload {
}
public void index(ResourceManager manager, boolean isCollection) {
public int index(ResourceManager manager, boolean isCollection) {
BulkRequest request = new BulkRequest();
int esResponseCode = 0;
while (manager.hasNext()){
try {
@ -125,24 +126,18 @@ public class BulkUpload {
.source(ace.toJson(),XContentType.JSON));
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse!=null) {
esResponseCode = bulkResponse.status().getStatus();
log.info("Indexing to ES completed with status: " + bulkResponse.status());
if (bulkResponse.hasFailures()) {
log.error("FailureMessage: " + bulkResponse.buildFailureMessage());
}
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
} catch (Exception e) {
log.error("Indexing "+e.getMessage());
return -1;
}
}
return esResponseCode;
}
}

18
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java
View File

@ -64,18 +64,18 @@ public class RunSPARQLQueryService {
return null;
final String selectQueryTemplate = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId);
recordIds.forEach(recordId -> {
executeQueryGraph(selectQueryTemplate, recordId, isCollection);
int operationResult = executeQueryGraph(selectQueryTemplate, recordId, isCollection);
log.info("Indexing final code: "+Integer.toString(operationResult));
});
return "ok";
}
private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){
log.debug("Retrieving "+recordId+" - isCollection:"+isCollection );
private int executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){
log.info("Retrieving "+recordId+" - isCollection:"+isCollection );
String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">");
openConnection();
StringWriter recordWriter = null;
Model resultsModel = null;
String jsonRecord = null;
try {
// log.debug("Started at: "+Calendar.getInstance().getTime().toString());
GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query);
@ -83,9 +83,9 @@ public class RunSPARQLQueryService {
resultsModel = QueryResults.asModel(graphQueryResult);
graphQueryResult.close();
// log.debug("Finished at: "+Calendar.getInstance().getTime().toString());
log.debug("Statements retrieved: " + resultsModel.size());
log.info("Statements retrieved: " + resultsModel.size());
if (resultsModel.size()==0) {
return "noresult";
return -2;
}
recordWriter = new StringWriter();
RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter);
@ -95,16 +95,16 @@ public class RunSPARQLQueryService {
}
parser.parse(recordWriter.toString());
resourceManager.manage(parser);
bulkUpload.index(resourceManager, isCollection);
return bulkUpload.index(resourceManager, isCollection);
} catch(Exception e){
e.printStackTrace();
log.error(e);
return -1;
} finally{
closeConnection();
if (resultsModel!=null) {
resultsModel.clear();
}
}
return jsonRecord;
}
public ParseRDFJSON getParser() {

1
dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql
View File

@ -14,6 +14,7 @@ INSERT {
?record aocat:has_ARIADNE_subject ?archeologicalResourceType .
}
}
USING <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::271>
WHERE {
?record aocat:is_part_of ?collection .
?collection aocat:was_issued ?issued .

15
dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/insert_ads_ariadne_subject.sparql Normal file
View File

@ -0,0 +1,15 @@
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::271> {
?collection aocat:has_ARIADNE_subject ?archeologicalResourceType .
?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .
}
}
WHERE {
?collection rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
?collection aocat:has_ARIADNE_subject ?archeologicalResourceType .
?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .
?collection aocat:has_original_id "1000271" .
}

6
dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java
View File

@ -55,9 +55,9 @@ public class GraphDbReaderAndESIndexTest {
runSPQRLQuery.setBulkUpload(bulkUpload);
String recordId;
final ClassPathResource queryTemplateResource;
boolean testRecord = false;
boolean testRecord = true;
if (testRecord) {
recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0320AA03-BED9-3F2B-AF5D-E31EE50CAE97";
recordId = "https://ariadne-infrastructure.eu/aocat/Resource/10DB6647-5D40-397A-8434-8CED55B2FC29";
queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
}
else {
@ -66,7 +66,7 @@ public class GraphDbReaderAndESIndexTest {
}
String datasource = "ads";
String collectionId = "271";
String collectionId = "270";
List<String> recordIds = Arrays.asList(recordId);
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
if (testRecord) {