From 3b95f932f35f4abb79c1471eea599834c8794a7f Mon Sep 17 00:00:00 2001 From: Katerina Date: Mon, 17 Jul 2023 12:33:19 +0300 Subject: [PATCH] Fixed bug on storing validationIssues --- .../controllers/ValidationController.java | 2 +- .../validatorapi/processors/XmlProcessor.java | 27 ++++++++--- .../validatorapi/routes/OaiPmhRoute.java | 45 ++++++++++--------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index ef88b80..2ef0bc9 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -213,7 +213,7 @@ public class ValidationController { RouteBuilder oaiPmhRouteBuilder = new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix , - profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository); + profile, validationJob, numberOfRecords); /* RouteBuilder oaiPmhRouteBuilder = diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java index 961e960..b782390 100644 --- a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java +++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java @@ -27,6 +27,7 @@ public class XmlProcessor implements Processor { // private final ValidationIssueRepository validationIssueRepository; private final long maxNumberOfRecords; private int processedRecords; + private String status = "SUCCESS"; // private int resultSum; private int scoreSum; @@ -47,6 +48,7 @@ public class XmlProcessor implements Processor { final DocumentBuilder documentBuilder = dbf.newDocumentBuilder(); Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml))); String recordUrl = extractRecordUrl(doc, "identifier"); + System.out.println("Identifier: " + recordUrl); List validationRuleResults = new ArrayList<>(); List validationIssues = new ArrayList<>(); @@ -62,22 +64,24 @@ public class XmlProcessor implements Processor { constructValidationRuleResult(validationRuleResult, validationJob.id, profile.name(), recordUrl, ruleName, profile, engineResult); + if (validationRuleResult.status.equals("FAILURE") || validationRuleResult.internalError!=null) { + this.status = "FAILURE"; + } if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) { validationRuleResult.hasErrors = true; for (String error:engineResult.errors()) { - validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, error,"ERROR")); + validationIssues.add(new ValidationIssue(validationJob.id, ruleName, recordUrl, error,"ERROR")); } } if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) { validationRuleResult.hasWarnings = true; for (String warning:engineResult.warnings()) { - validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, warning, "WARNING")); + validationIssues.add(new ValidationIssue(validationJob.id, ruleName, recordUrl, warning, "WARNING")); } } - //validationResultRepository.save(validationRuleResult); validationRuleResults.add(validationRuleResult); scoreSum += engineResult.score(); } @@ -87,14 +91,23 @@ public class XmlProcessor implements Processor { processedRecords++; validationJob.recordsTested = processedRecords; validationJob.score = scoreSum/processedRecords; - Map resultsMap = new HashMap(); - resultsMap.put("results", validationRuleResults); - resultsMap.put("issues", validationIssues); + resultsMap.put("issues", validationRuleResults); + resultsMap.put("results", validationIssues); exchange.getIn().setBody(resultsMap, Map.class); - if (processedRecords == maxNumberOfRecords) { + // System.out.println("Validation Issues: " + validationIssues); + + if (processedRecords > maxNumberOfRecords) { exchange.getIn().setHeader("MyHeader", "stop"); + exchange.getIn().setHeader("validationId", validationJob.id); + exchange.getIn().setHeader("baseUrl", validationJob.baseUrl); + exchange.getIn().setHeader("startDate", validationJob.startDate); + exchange.getIn().setHeader("guidelines", validationJob.guidelines); + exchange.getIn().setHeader("status", status); + exchange.getIn().setHeader("recordsTested", processedRecords-1); + exchange.getIn().setHeader("maxNumberOfRecords", maxNumberOfRecords); + exchange.getIn().setHeader("score", scoreSum/processedRecords); } } diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java index 7237eb1..12cd916 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java @@ -6,9 +6,6 @@ import eu.dnetlib.validatorapi.entities.ValidationJob; import eu.dnetlib.validatorapi.entities.ValidationRuleResult; import eu.dnetlib.validatorapi.processors.DataBaseProcessor; import eu.dnetlib.validatorapi.processors.XmlProcessor; -import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; -import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; -import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.apache.camel.builder.RouteBuilder; @@ -23,23 +20,21 @@ public class OaiPmhRoute extends RouteBuilder { private AbstractOpenAireProfile profile; private long maxNumberOfRecords = 0; private ValidationJob validationJob; - private final ValidationJobRepository validationJobRepository; - private final ValidationIssueRepository validationIssueRepository; - private final ValidationResultRepository validationResultRepository; + //private final ValidationJobRepository validationJobRepository; + //private final ValidationIssueRepository validationIssueRepository; + //private final ValidationResultRepository validationResultRepository; CountDownLatch latch; public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob, - long maxNumberOfRecords, final ValidationJobRepository validationJobRepository, - final ValidationIssueRepository validationIssueRepository, - final ValidationResultRepository validationResultRepository) { + long maxNumberOfRecords) { this.oaiEndpoint = oaiEndpoint; this.validationJob = validationJob; this.profile = profile; this.maxNumberOfRecords = maxNumberOfRecords; - this.validationJobRepository = validationJobRepository; - this.validationIssueRepository = validationIssueRepository; - this.validationResultRepository = validationResultRepository; + // this.validationJobRepository = validationJobRepository; + // this.validationIssueRepository = validationIssueRepository; + // this.validationResultRepository = validationResultRepository; this.latch = latch; } @@ -53,21 +48,29 @@ public class OaiPmhRoute extends RouteBuilder { .split(xpath("//*[local-name()='record']")) .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords)) .choice() - .when(simple("${body[results]} && ${header.MyHeader} != 'stop'")) - .split(simple("${body[results]}")) - .to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true") - .endChoice() .when(simple("${body[issues]} && ${header.MyHeader} != 'stop'")) - .split(simple("${body[issues]}")) - .to("jpa:"+ ValidationIssue.class.getName()+ "?usePersist=true") + .log("HERE!!!") + .split(simple("${body[issues]}")) + .to("jpa:"+ ValidationRuleResult.class.getName()+ "?usePersist=true") .endChoice() + .end() + .choice() + .when(simple("${body[results]} && ${header.MyHeader} != 'stop'")) + .log("THERE") + .split(simple("${body[results]}")) + .to("jpa:" + ValidationIssue.class.getName() + "?usePersist=true") + .endChoice() + .end() + .choice() .when(header("MyHeader").isEqualTo("stop")) - //.to("direct:saveToDatabase") + .to("direct:saveToDatabase") .to("controlbus:route?routeId="+date+"&action=stop") - .endChoice(); + .endChoice() + .end(); + from("direct:saveToDatabase") - .process(new DataBaseProcessor(validationJob, validationResultRepository)) + .process(new DataBaseProcessor()) .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true"); /*from("timer://myTimer?fixedRate=true&period=60000")