collection is now indexed before records, if there is an error on collection indexing, wf failed immediately; enriched query for 397 collection

pull/2/head
Enrico Ottonello 4 years ago
parent 12bbcdd8be
commit 814c908663

@ -347,13 +347,7 @@ public class GraphDBClient {
runSPQRLQuery.setParser(parseRDFJSON);
runSPQRLQuery.setResourceManager(resourceManager);
runSPQRLQuery.setBulkUpload(bulkUpload);
List<String> recordIds = runSPQRLQuery.selectRecordIds(datasource, collectionId);
log.info(String.format("Found %d individual resources to index for datasource %s - %s", recordIds.size(), datasource, collectionId));
if(!recordIds.isEmpty()) {
final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
recordsIndexReport = runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false);
}
List<String> collectionResourceIds = runSPQRLQuery.selectCollectionIds(datasource, collectionId);
log.info(String.format("Found %d collections to index for datasource %s - %s", collectionResourceIds.size(), datasource, collectionId));
if(!collectionResourceIds.isEmpty()) {
@ -361,6 +355,15 @@ public class GraphDBClient {
String selectCollectionTemplate = IOUtils.toString(selectCollectionTemplateRes.getInputStream(), StandardCharsets.UTF_8.name());
collectionIndexReport = runSPQRLQuery.executeMultipleQueryGraph(selectCollectionTemplate, collectionResourceIds, datasource, collectionId, true);
}
List<String> recordIds = runSPQRLQuery.selectRecordIds(datasource, collectionId);
log.info(String.format("Found %d individual resources to index for datasource %s - %s", recordIds.size(), datasource, collectionId));
if(!recordIds.isEmpty()) {
final ClassPathResource queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");
String queryTemplate = IOUtils.toString(queryTemplateResource.getInputStream(), StandardCharsets.UTF_8.name());
recordsIndexReport = runSPQRLQuery.executeMultipleQueryGraph(queryTemplate, recordIds, datasource, collectionId, false);
}
}catch(Throwable e){
log.error(e);
throw new AriadnePlusPublisherException(e);

@ -1,6 +1,7 @@
package eu.dnetlib.ariadneplus.reader;
import eu.dnetlib.ariadneplus.elasticsearch.BulkUpload;
import eu.dnetlib.ariadneplus.publisher.AriadnePlusPublisherException;
import eu.dnetlib.ariadneplus.reader.json.ParseRDFJSON;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -60,7 +61,8 @@ public class RunSPARQLQueryService {
manager.shutDown();
}
public String executeMultipleQueryGraph(String queryTemplate, List<String> recordIds, String datasource, String collectionId, boolean isCollection){
public String executeMultipleQueryGraph(String queryTemplate, List<String> recordIds, String datasource, String collectionId, boolean isCollection)
throws AriadnePlusPublisherException {
if (queryTemplate==null)
return null;
final String selectQueryTemplate = queryTemplate.replaceAll("%datasource", datasource).replaceAll("%collectionId", collectionId);
@ -74,6 +76,9 @@ public class RunSPARQLQueryService {
log.debug(" "+operationResult);
if (operationResult!=200) {
log.error(recordId + " error_code: "+ operationResult);
if (isCollection) {
throw new AriadnePlusPublisherException("Collection indexing problem - errcode: "+operationResult);
}
int currentErrorsCount = errorCodesCount.get(0).intValue();
currentErrorsCount+=1;
errorCodesCount.set(0, new Integer(currentErrorsCount));
@ -85,7 +90,7 @@ public class RunSPARQLQueryService {
}
int counterValue = counter.get(0).intValue();
String curReport = null;
if ((counterValue % 1000) == 0) {
if ((counterValue>0) && (counterValue % 1000) == 0) {
curReport = "Current analyzed records: "+counterValue+" Current indexed records: "+ successCodesCount.get(0).intValue() +
" , " + "Current errors: "+ errorCodesCount.get(0).intValue();
log.info(curReport);

@ -0,0 +1,140 @@
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397> {
?collection aocat:has_ARIADNE_subject ?archeologicalResourceType .
?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .
?archeologicalResourceType rdfs:label ?archeologicalResourceTypeName .
}
}
WHERE {
?collection rdf:type <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/AO_Collection> .
?collection aocat:has_ARIADNE_subject ?archeologicalResourceType .
?archeologicalResourceType skos:prefLabel ?archeologicalResourceTypeName .
?collection aocat:has_original_id "1000397" .
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX ariadneplus: <https://ariadne-infrastructure.eu/aocat/>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397> {
?record aocat:was_issued ?issued .
?record aocat:was_modified ?modified .
?record aocat:has_contributor ?contributor .
?record aocat:has_responsible ?legalResponsible .
?record aocat:has_owner ?owner .
?record aocat:has_publisher ?publisher .
?record aocat:has_access_rights ?accessRights .
?record aocat:has_ARIADNE_subject ?archeologicalResourceType .
}
}
USING <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397>
WHERE {
?record aocat:is_part_of ?collection .
?collection aocat:was_issued ?issued .
?collection aocat:was_modified ?modified .
?collection aocat:has_contributor ?contributor .
?collection aocat:has_responsible ?legalResponsible .
?collection aocat:has_owner ?owner .
?collection aocat:has_publisher ?publisher .
?collection aocat:has_access_rights ?accessRights .
?collection aocat:has_ARIADNE_subject ?archeologicalResourceType .
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397> {
?s aocat:has_native_subject <https://ariadne-infrastructure.eu/aocat/Concept/ads/NOT_PROVIDED> .
<https://ariadne-infrastructure.eu/aocat/Concept/ads/NOT_PROVIDED> skos:prefLabel "Not provided" .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397> {
?s rdf:type aocat:AO_Individual_Data_Resource .
MINUS {
?s rdf:type aocat:AO_Individual_Data_Resource .
?s aocat:has_native_subject ?ns .
}
}
};
INSERT {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397> {
?s aocat:has_title "Title not provided" .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397> {
?s rdf:type aocat:AO_Individual_Data_Resource .
MINUS {
?s rdf:type aocat:AO_Individual_Data_Resource .
?s aocat:has_title ?title .
}
}
};
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/ariadneplus::ads::aatplus> {
?record aocat:has_derived_subject ?aat .
}
}
USING <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::aat>
USING <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397>
WHERE {
{
?record aocat:has_native_subject ?native_subject .
?native_subject skos:exactMatch ?aat .
}
union
{
?record aocat:has_native_subject ?native_subject .
?native_subject skos:broadMatch ?aat .
}
union
{
?record aocat:has_native_subject ?native_subject .
?native_subject skos:closeMatch ?aat .
}
union
{
?record aocat:has_native_subject ?native_subject .
?native_subject skos:narrowMatch ?aat .
}
};
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX aocat: <https://www.ariadne-infrastructure.eu/resource/ao/cat/1.1/>
PREFIX time: <http://www.w3.org/2006/time#>
INSERT {
GRAPH <https://ariadne-infrastructure.eu/ariadneplus::ads::periodoplus> {
?temporal aocat:has_period ?periodO .
?temporal aocat:from ?temporalFrom .
?temporal aocat:until ?temporalUntil .
}
}
WHERE {
GRAPH <https://ariadne-infrastructure.eu/api_________::ariadne_plus::ads::397> {
?temporal aocat:has_native_period ?native_period .
?native_period skos:prefLabel ?native_label .
optional {
GRAPH <https://ariadne-infrastructure.eu/ariadneplus::ads::periodo> {
?periodO skos:altLabel ?native_label .
?periodO skos:inScheme <http://n2t.net/ark:/99152/p0kh9ds> .
?periodO time:intervalStartedBy ?intervalStartedBy .
?intervalStartedBy skos:prefLabel ?temporalFrom .
?periodO time:intervalFinishedBy ?intervalFinishedBy .
?intervalFinishedBy skos:prefLabel ?temporalUntil .
}
}
}
};
Loading…
Cancel
Save