MaDgIK
/
uoa-validator-api
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed bug on storing validationIssues

This commit is contained in:
Katerina 2023-07-17 12:33:19 +03:00
parent 7b5a9cad3d
commit 3b95f932f3
3 changed files with 45 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
View File

@ -213,7 +213,7 @@ public class ValidationController {
RouteBuilder oaiPmhRouteBuilder = RouteBuilder oaiPmhRouteBuilder =
new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix , new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository); profile, validationJob, numberOfRecords);
/* /*
RouteBuilder oaiPmhRouteBuilder = RouteBuilder oaiPmhRouteBuilder =

27
src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java
View File

@ -27,6 +27,7 @@ public class XmlProcessor implements Processor {
// private final ValidationIssueRepository validationIssueRepository; // private final ValidationIssueRepository validationIssueRepository;
private final long maxNumberOfRecords; private final long maxNumberOfRecords;
private int processedRecords; private int processedRecords;
private String status = "SUCCESS";
// private int resultSum; // private int resultSum;
private int scoreSum; private int scoreSum;
@ -47,6 +48,7 @@ public class XmlProcessor implements Processor {
final DocumentBuilder documentBuilder = dbf.newDocumentBuilder(); final DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml))); Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml)));
String recordUrl = extractRecordUrl(doc, "identifier"); String recordUrl = extractRecordUrl(doc, "identifier");
System.out.println("Identifier: " + recordUrl);
List<ValidationRuleResult> validationRuleResults = new ArrayList<>(); List<ValidationRuleResult> validationRuleResults = new ArrayList<>();
List<ValidationIssue> validationIssues = new ArrayList<>(); List<ValidationIssue> validationIssues = new ArrayList<>();
@ -62,22 +64,24 @@ public class XmlProcessor implements Processor {
constructValidationRuleResult(validationRuleResult, validationJob.id, profile.name(), recordUrl, constructValidationRuleResult(validationRuleResult, validationJob.id, profile.name(), recordUrl,
ruleName, profile, engineResult); ruleName, profile, engineResult);
if (validationRuleResult.status.equals("FAILURE") || validationRuleResult.internalError!=null) {
this.status = "FAILURE";
}
if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) { if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) {
validationRuleResult.hasErrors = true; validationRuleResult.hasErrors = true;
for (String error:engineResult.errors()) { for (String error:engineResult.errors()) {
validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, error,"ERROR")); validationIssues.add(new ValidationIssue(validationJob.id, ruleName, recordUrl, error,"ERROR"));
} }
} }
if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) { if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) {
validationRuleResult.hasWarnings = true; validationRuleResult.hasWarnings = true;
for (String warning:engineResult.warnings()) { for (String warning:engineResult.warnings()) {
validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, warning, "WARNING")); validationIssues.add(new ValidationIssue(validationJob.id, ruleName, recordUrl, warning, "WARNING"));
} }
} }
//validationResultRepository.save(validationRuleResult);
validationRuleResults.add(validationRuleResult); validationRuleResults.add(validationRuleResult);
scoreSum += engineResult.score(); scoreSum += engineResult.score();
} }
@ -87,14 +91,23 @@ public class XmlProcessor implements Processor {
processedRecords++; processedRecords++;
validationJob.recordsTested = processedRecords; validationJob.recordsTested = processedRecords;
validationJob.score = scoreSum/processedRecords; validationJob.score = scoreSum/processedRecords;
Map<String, List> resultsMap = new HashMap(); Map<String, List> resultsMap = new HashMap();
resultsMap.put("results", validationRuleResults); resultsMap.put("issues", validationRuleResults);
resultsMap.put("issues", validationIssues); resultsMap.put("results", validationIssues);
exchange.getIn().setBody(resultsMap, Map.class); exchange.getIn().setBody(resultsMap, Map.class);
if (processedRecords == maxNumberOfRecords) { // System.out.println("Validation Issues: " + validationIssues);
if (processedRecords > maxNumberOfRecords) {
exchange.getIn().setHeader("MyHeader", "stop"); exchange.getIn().setHeader("MyHeader", "stop");
exchange.getIn().setHeader("validationId", validationJob.id);
exchange.getIn().setHeader("baseUrl", validationJob.baseUrl);
exchange.getIn().setHeader("startDate", validationJob.startDate);
exchange.getIn().setHeader("guidelines", validationJob.guidelines);
exchange.getIn().setHeader("status", status);
exchange.getIn().setHeader("recordsTested", processedRecords-1);
exchange.getIn().setHeader("maxNumberOfRecords", maxNumberOfRecords);
exchange.getIn().setHeader("score", scoreSum/processedRecords);
} }
} }

45
src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java
View File

@ -6,9 +6,6 @@ import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.entities.ValidationRuleResult; import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
import eu.dnetlib.validatorapi.processors.DataBaseProcessor; import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
import eu.dnetlib.validatorapi.processors.XmlProcessor; import eu.dnetlib.validatorapi.processors.XmlProcessor;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange; import org.apache.camel.Exchange;
import org.apache.camel.Processor; import org.apache.camel.Processor;
import org.apache.camel.builder.RouteBuilder; import org.apache.camel.builder.RouteBuilder;
@ -23,23 +20,21 @@ public class OaiPmhRoute extends RouteBuilder {
private AbstractOpenAireProfile profile; private AbstractOpenAireProfile profile;
private long maxNumberOfRecords = 0; private long maxNumberOfRecords = 0;
private ValidationJob validationJob; private ValidationJob validationJob;
private final ValidationJobRepository validationJobRepository; //private final ValidationJobRepository validationJobRepository;
private final ValidationIssueRepository validationIssueRepository; //private final ValidationIssueRepository validationIssueRepository;
private final ValidationResultRepository validationResultRepository; //private final ValidationResultRepository validationResultRepository;
CountDownLatch latch; CountDownLatch latch;
public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob, public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob,
long maxNumberOfRecords, final ValidationJobRepository validationJobRepository, long maxNumberOfRecords) {
final ValidationIssueRepository validationIssueRepository,
final ValidationResultRepository validationResultRepository) {
this.oaiEndpoint = oaiEndpoint; this.oaiEndpoint = oaiEndpoint;
this.validationJob = validationJob; this.validationJob = validationJob;
this.profile = profile; this.profile = profile;
this.maxNumberOfRecords = maxNumberOfRecords; this.maxNumberOfRecords = maxNumberOfRecords;
this.validationJobRepository = validationJobRepository; // this.validationJobRepository = validationJobRepository;
this.validationIssueRepository = validationIssueRepository; // this.validationIssueRepository = validationIssueRepository;
this.validationResultRepository = validationResultRepository; // this.validationResultRepository = validationResultRepository;
this.latch = latch; this.latch = latch;
} }
@ -53,21 +48,29 @@ public class OaiPmhRoute extends RouteBuilder {
.split(xpath("//*[local-name()='record']")) .split(xpath("//*[local-name()='record']"))
.process(new XmlProcessor(profile, validationJob, maxNumberOfRecords)) .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords))
.choice() .choice()
.when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
.split(simple("${body[results]}"))
.to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true")
.endChoice()
.when(simple("${body[issues]} && ${header.MyHeader} != 'stop'")) .when(simple("${body[issues]} && ${header.MyHeader} != 'stop'"))
.split(simple("${body[issues]}")) .log("HERE!!!")
.to("jpa:"+ ValidationIssue.class.getName()+ "?usePersist=true") .split(simple("${body[issues]}"))
.to("jpa:"+ ValidationRuleResult.class.getName()+ "?usePersist=true")
.endChoice() .endChoice()
.end()
.choice()
.when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
.log("THERE")
.split(simple("${body[results]}"))
.to("jpa:" + ValidationIssue.class.getName() + "?usePersist=true")
.endChoice()
.end()
.choice()
.when(header("MyHeader").isEqualTo("stop")) .when(header("MyHeader").isEqualTo("stop"))
//.to("direct:saveToDatabase") .to("direct:saveToDatabase")
.to("controlbus:route?routeId="+date+"&action=stop") .to("controlbus:route?routeId="+date+"&action=stop")
.endChoice(); .endChoice()
.end();
from("direct:saveToDatabase") from("direct:saveToDatabase")
.process(new DataBaseProcessor(validationJob, validationResultRepository)) .process(new DataBaseProcessor())
.to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true"); .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true");
/*from("timer://myTimer?fixedRate=true&period=60000") /*from("timer://myTimer?fixedRate=true&period=60000")