From 5bb65a88540aa6f91804d5cd21e8712a3aed62bf Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Wed, 22 Jul 2020 23:13:47 +0200 Subject: [PATCH] added response code from es; added sparql insert for copying ariadne_subject value into collection for ads; fix sparql insert for copying collection data into records, collection id condition is needed --- .../ariadneplus/elasticsearch/BulkUpload.java | 29 ++++++++----------- .../reader/RunSPARQLQueryService.java | 18 ++++++------ .../sparql/collection_to_records_data.sparql | 1 + .../sparql/insert_ads_ariadne_subject.sparql | 15 ++++++++++ .../GraphDbReaderAndESIndexTest.java | 6 ++-- 5 files changed, 40 insertions(+), 29 deletions(-) create mode 100644 dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/insert_ads_ariadne_subject.sparql diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index f330b96..9c69ea2 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -49,8 +49,9 @@ public class BulkUpload { } - public void index(ResourceManager manager, boolean isCollection) { + public int index(ResourceManager manager, boolean isCollection) { BulkRequest request = new BulkRequest(); + int esResponseCode = 0; while (manager.hasNext()){ try { @@ -125,24 +126,18 @@ public class BulkUpload { .source(ace.toJson(),XContentType.JSON)); BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); - log.info("Indexing to ES completed with status: "+bulkResponse.status()); - if (bulkResponse.hasFailures()) { - log.error("FailureMessage: "+bulkResponse.buildFailureMessage()); + if (bulkResponse!=null) { + esResponseCode = bulkResponse.status().getStatus(); + log.info("Indexing to ES completed with status: " + bulkResponse.status()); + if (bulkResponse.hasFailures()) { + log.error("FailureMessage: " + bulkResponse.buildFailureMessage()); + } } - - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } catch (NoSuchMethodException e) { - e.printStackTrace(); - } catch (InstantiationException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (InvocationTargetException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); + } catch (Exception e) { + log.error("Indexing "+e.getMessage()); + return -1; } } + return esResponseCode; } } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index 075db7d..68ebefb 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -64,18 +64,18 @@ public class RunSPARQLQueryService { return null; final String selectQueryTemplate = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId); recordIds.forEach(recordId -> { - executeQueryGraph(selectQueryTemplate, recordId, isCollection); + int operationResult = executeQueryGraph(selectQueryTemplate, recordId, isCollection); + log.info("Indexing final code: "+Integer.toString(operationResult)); }); return "ok"; } - private String executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){ - log.debug("Retrieving "+recordId+" - isCollection:"+isCollection ); + private int executeQueryGraph(String selectQueryTemplate, String recordId, boolean isCollection){ + log.info("Retrieving "+recordId+" - isCollection:"+isCollection ); String query = selectQueryTemplate.replaceAll("%record", "<"+recordId+">"); openConnection(); StringWriter recordWriter = null; Model resultsModel = null; - String jsonRecord = null; try { // log.debug("Started at: "+Calendar.getInstance().getTime().toString()); GraphQuery graphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, query); @@ -83,9 +83,9 @@ public class RunSPARQLQueryService { resultsModel = QueryResults.asModel(graphQueryResult); graphQueryResult.close(); // log.debug("Finished at: "+Calendar.getInstance().getTime().toString()); - log.debug("Statements retrieved: " + resultsModel.size()); + log.info("Statements retrieved: " + resultsModel.size()); if (resultsModel.size()==0) { - return "noresult"; + return -2; } recordWriter = new StringWriter(); RDFWriter rdfRecordWriter = Rio.createWriter(RDFFormat.RDFJSON, recordWriter); @@ -95,16 +95,16 @@ public class RunSPARQLQueryService { } parser.parse(recordWriter.toString()); resourceManager.manage(parser); - bulkUpload.index(resourceManager, isCollection); + return bulkUpload.index(resourceManager, isCollection); } catch(Exception e){ - e.printStackTrace(); + log.error(e); + return -1; } finally{ closeConnection(); if (resultsModel!=null) { resultsModel.clear(); } } - return jsonRecord; } public ParseRDFJSON getParser() { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql index 40d936f..7682e60 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/collection_to_records_data.sparql @@ -14,6 +14,7 @@ INSERT { ?record aocat:has_ARIADNE_subject ?archeologicalResourceType . } } +USING WHERE { ?record aocat:is_part_of ?collection . ?collection aocat:was_issued ?issued . diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/insert_ads_ariadne_subject.sparql b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/insert_ads_ariadne_subject.sparql new file mode 100644 index 0000000..81716ca --- /dev/null +++ b/dnet-ariadneplus-graphdb-publisher/src/main/resources/eu/dnetlib/ariadneplus/sparql/insert_ads_ariadne_subject.sparql @@ -0,0 +1,15 @@ +PREFIX aocat: +PREFIX skos: +PREFIX rdf: +INSERT { + GRAPH { + ?collection aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + } +} +WHERE { + ?collection rdf:type . + ?collection aocat:has_ARIADNE_subject ?archeologicalResourceType . + ?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName . + ?collection aocat:has_original_id "1000271" . +} \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index 9414f0f..918e17a 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -55,9 +55,9 @@ public class GraphDbReaderAndESIndexTest { runSPQRLQuery.setBulkUpload(bulkUpload); String recordId; final ClassPathResource queryTemplateResource; - boolean testRecord = false; + boolean testRecord = true; if (testRecord) { - recordId = "https://ariadne-infrastructure.eu/aocat/Resource/0320AA03-BED9-3F2B-AF5D-E31EE50CAE97"; + recordId = "https://ariadne-infrastructure.eu/aocat/Resource/10DB6647-5D40-397A-8434-8CED55B2FC29"; queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql"); } else { @@ -66,7 +66,7 @@ public class GraphDbReaderAndESIndexTest { } String datasource = "ads"; - String collectionId = "271"; + String collectionId = "270"; List recordIds = Arrays.asList(recordId); String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name()); if (testRecord) {