From 004b166b7329a830c8cf0f1435cb4fccfae0e1b3 Mon Sep 17 00:00:00 2001 From: Katerina Date: Tue, 13 Jun 2023 15:57:39 +0300 Subject: [PATCH] Added separate processor to save in db + refined the APIs for the job and the error/warnings + added repository method for the status of a job --- .../controllers/ReportController.java | 38 +++++++++---------- .../controllers/ValidationController.java | 27 ++++++------- .../validatorapi/entities/ValidationJob.java | 14 +++++-- .../processors/DataBaseProcessor.java | 32 ++++++++++++++++ .../processors/StopRouteProcessor.java | 4 -- .../validatorapi/processors/XmlProcessor.java | 12 ++++-- .../ValidationIssueRepository.java | 20 ++++++++-- .../ValidationResultRepository.java | 6 +++ .../validatorapi/routes/OaiPmhRoute.java | 14 ++++++- 9 files changed, 115 insertions(+), 52 deletions(-) create mode 100644 src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java delete mode 100644 src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java index 117f253..bac6066 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java @@ -25,37 +25,37 @@ public class ReportController { this.validationJobRepository = validationJobRepository; this.validationResultRepository = validationResultRepository; this.validationIssueRepository = validationIssueRepository; - } + } + + @RequestMapping(value={"getJobResult"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public ValidationJob getJobResults(@RequestParam(name = "jobId") int jobId){ + Optional validationJob = validationJobRepository.findById(jobId); + return (ValidationJob) validationJob.orElse(null); + } - @RequestMapping(value={"getJobResult"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public ValidationJob getJobResults(@RequestParam(name = "jobId") int jobId){ - Optional validationJob = validationJobRepository.findById(jobId); - return (ValidationJob) validationJob.orElse(null); - } @RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public List getSummaryJobResults(@RequestParam(name = "jobId") int jobId){ return validationResultRepository.getSummaryResult(jobId); } + @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public Map> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ + List resultList = validationIssueRepository.getAllErrorsRecordUrlsByRuleName(jobId, ruleName); + return extractRecordsGroupedByRule(resultList); + } + + @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public Map> getWarningsReport(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ + List resultList = validationIssueRepository.getAllWarningsRecordUrlsByRuleName(jobId, ruleName); + return extractRecordsGroupedByRule(resultList); + } + @RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public List getRecordsByRule(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ return validationIssueRepository.getErrors(jobId, ruleName); } - @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public Map> getErrors(@RequestParam(name = "jobId") int jobId){ - List resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId); - return extractRecordsGroupedByRule(resultList); - } - - @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public Map> getWarningsReport(@RequestParam(name = "jobId") int jobId, - @RequestParam(name = "ruleName") String ruleName){ - List resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId); - return extractRecordsGroupedByRule(resultList); - } - private Map> extractRecordsGroupedByRule(List resultList) { Map> recordUrlsByIssueText = new HashMap<>(); diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index cda9171..44e09b1 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -129,13 +129,13 @@ public class ValidationController { System.out.println(record++); } } - validationJob.status = "COMPLETED"; + validationJob.progress = "COMPLETED"; } catch (Exception e) { log.error("Validation job stopped unexpectedly." + e.getMessage()); System.out.println("ERROR " + e.getMessage()); - validationJob.status = "STOPPED"; + validationJob.progress = "STOPPED"; } finally { validationJob.endDate = new Date(); @@ -143,6 +143,7 @@ public class ValidationController { validationJob.recordsTested = record; validationJob.score = resultSum / record; //TODO uncomment + log.info("Saving validation job " + validationJob.recordsTested); validationJobRepository.save(validationJob); } @@ -197,11 +198,9 @@ public class ValidationController { ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords); validationJob.guidelines = profile.name(); - validationJob.status = "IN_PROGRESS"; validationJobRepository.save(validationJob); log.info("Initial validation job id "+ validationJob.id); - int record = 0; double resultSum = 0; @@ -211,24 +210,21 @@ public class ValidationController { RouteBuilder oaiPmhRouteBuilder = new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix , - profile, validationJob, numberOfRecords, validationIssueRepository, validationResultRepository); + profile, validationJob, numberOfRecords, validationJobRepository, + validationIssueRepository, validationResultRepository); camelContext.addRoutes(oaiPmhRouteBuilder); - validationJob.status = "COMPLETED"; + validationJob.progress = "COMPLETED"; } catch (Exception e) { log.error("Validation job stopped unexpectedly." + e.getMessage()); System.out.println("ERROR " + e.getMessage()); - validationJob.status = "STOPPED"; - - } finally { + validationJob.progress = "STOPPED"; validationJob.endDate = new Date(); - log.info("Final validation job "+ validationJob.id); - validationJob.recordsTested = record; - validationJob.score = resultSum / record; + validationJob.score = resultSum / validationJob.recordsTested; + validationJob.status = validationResultRepository.getStatus(validationJob.id); validationJobRepository.save(validationJob); } - } @@ -425,12 +421,13 @@ public class ValidationController { } record++; } - validationJob.status = "COMPLETED"; + + validationJob.progress = "COMPLETED"; } catch (Exception e) { log.error("Validation job stopped unexpectedly." + e.getMessage()); - validationJob.status = "STOPPED"; + validationJob.progress = "STOPPED"; } finally { validationJob.endDate = new Date(); diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java index ae753bb..e12be6b 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java @@ -8,7 +8,7 @@ import java.util.Date; public class ValidationJob { @GeneratedValue(strategy = GenerationType.IDENTITY) @Id - public int id; //not in use for now + public int id; @Column(name = "base_url") public String baseUrl; @@ -24,8 +24,10 @@ public class ValidationJob { @Column(name = "records_tested") public int recordsTested; - @Column(name="duration") - public String status; //stopped, completed, in progress + @Column(name="progress") + public String progress; //stopped, completed, in progress + @Column(name="status") + public String status; //success, failure @Column(name="score") public double score; @@ -42,7 +44,7 @@ public class ValidationJob { @Override public String toString() { - return "ValidationJob {" + + return "ValidationJob{" + "id=" + id + ", baseUrl='" + baseUrl + '\'' + ", numberOfRecords=" + numberOfRecords + @@ -50,8 +52,12 @@ public class ValidationJob { ", startDate=" + startDate + ", endDate=" + endDate + ", recordsTested=" + recordsTested + + ", progress='" + progress + '\'' + ", status='" + status + '\'' + ", score=" + score + '}'; } + + + } diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java new file mode 100644 index 0000000..0aca156 --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java @@ -0,0 +1,32 @@ +package eu.dnetlib.validatorapi.processors; + +import eu.dnetlib.validatorapi.entities.ValidationJob; +import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; +import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; +import org.apache.camel.Exchange; +import org.apache.camel.Processor; + +import java.util.Date; + +public class DataBaseProcessor implements Processor { + private final ValidationJob validationJob; + private final ValidationJobRepository validationJobRepository; + private final ValidationResultRepository validationResultRepository; + + public DataBaseProcessor(ValidationJob validationJob, final ValidationJobRepository validationJobRepository, + final ValidationResultRepository validationResultRepository){ + this.validationJob = validationJob; + this.validationJobRepository = validationJobRepository; + this.validationResultRepository = validationResultRepository; + + } + + @Override + public void process(Exchange exchange) throws Exception { + validationJob.progress = "COMPLETED"; + validationJob.endDate = new Date(); + validationJob.status = validationResultRepository.getStatus(validationJob.id); + System.out.println(validationJob.status); + validationJobRepository.save(validationJob); + } +} diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java deleted file mode 100644 index 1c7b143..0000000 --- a/src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java +++ /dev/null @@ -1,4 +0,0 @@ -package eu.dnetlib.validatorapi.processors; - -public class StopRouteProcessor { -} diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java index bf5d9a8..a901394 100644 --- a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java +++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java @@ -22,14 +22,15 @@ import java.util.Map; public class XmlProcessor implements Processor { private final AbstractOpenAireProfile profile; - private final ValidationJob validationJob; + private ValidationJob validationJob; private final ValidationIssueRepository validationIssueRepository; private final ValidationResultRepository validationResultRepository; private final long maxNumberOfRecords; - private long processedRecords; + private int processedRecords; + private int resultSum; - public XmlProcessor(final AbstractOpenAireProfile profile, final ValidationJob validationJob, + public XmlProcessor(final AbstractOpenAireProfile profile, ValidationJob validationJob, final ValidationIssueRepository validationIssueRepository, final ValidationResultRepository validationResultRepository, final long maxNumberOfRecords){ @@ -64,13 +65,16 @@ public class XmlProcessor implements Processor { constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl, ruleName, profile, engineResult); validationResultRepository.save(validationRuleResult); + resultSum += engineResult.score(); saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR"); saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING"); } } processedRecords++; - exchange.setProperty("totalRecords", processedRecords); + validationJob.recordsTested = processedRecords; + validationJob.score = resultSum; + System.out.println("\n\n\nvalidation job: " + processedRecords); if (processedRecords > maxNumberOfRecords) { exchange.getIn().setHeader("MyHeader", "stop"); diff --git a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java index 6198041..7b7508a 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java @@ -26,14 +26,26 @@ public interface ValidationIssueRepository extends JpaRepository getAllErrorsRecordUrls(@Param("id") int validationJobId); + List getAllErrorsRecordUrlsByRuleName(@Param("id") int validationJobId, @Param("ruleName") String ruleName); @Query(value = "SELECT vi.issueText, vi.recordUrl " + "FROM ValidationIssue vi " + - "WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id)" + + "WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id AND vi.ruleName=:ruleName)" + "GROUP BY vi.issueText, vi.recordUrl") - List getAllWarningsRecordUrls(@Param("id") int validationJobId); + List getAllWarningsRecordUrlsByRuleName(@Param("id") int validationJobId, @Param("ruleName") String ruleName); + + @Query(value = "SELECT vi.ruleName, vi.issueText, vi.recordUrl " + + "FROM ValidationIssue vi " + + "WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" + + "GROUP BY vi.ruleName, vi.issueText, vi.recordUrl") + List getAllErrors(@Param("id") int validationJobId); + + @Query(value = "SELECT vi.ruleName, vi.issueText, vi.recordUrl " + + "FROM ValidationIssue vi " + + "WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id)" + + "GROUP BY vi.ruleName, vi.issueText, vi.recordUrl") + List getAllWarnings(@Param("id") int validationJobId); } diff --git a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java index 3f33ae4..46fe887 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java @@ -19,4 +19,10 @@ public interface ValidationResultRepository extends JpaRepository getSummaryResult(@Param("id") int validationJobId); + + @Query("SELECT CASE WHEN COUNT(vr) > 0 THEN 'FAILURE' ELSE 'SUCCESS' END " + + "FROM ValidationRuleResult vr " + + "WHERE vr.validationJobId = :id AND vr.status = 'FAILURE'") + public String getStatus(@Param("id") int validationJobId); + } diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java index efae172..4713d31 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java @@ -2,8 +2,10 @@ package eu.dnetlib.validatorapi.routes; import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile; import eu.dnetlib.validatorapi.entities.ValidationJob; +import eu.dnetlib.validatorapi.processors.DataBaseProcessor; import eu.dnetlib.validatorapi.processors.XmlProcessor; import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; +import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import org.apache.camel.Exchange; import org.apache.camel.Processor; @@ -19,18 +21,21 @@ public class OaiPmhRoute extends RouteBuilder { private AbstractOpenAireProfile profile; private long maxNumberOfRecords = 0; private ValidationJob validationJob; + private final ValidationJobRepository validationJobRepository; private final ValidationIssueRepository validationIssueRepository; private final ValidationResultRepository validationResultRepository; CountDownLatch latch; public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob, - long maxNumberOfRecords, final ValidationIssueRepository validationIssueRepository, + long maxNumberOfRecords, final ValidationJobRepository validationJobRepository, + final ValidationIssueRepository validationIssueRepository, final ValidationResultRepository validationResultRepository) { this.oaiEndpoint = oaiEndpoint; this.validationJob = validationJob; this.profile = profile; this.maxNumberOfRecords = maxNumberOfRecords; + this.validationJobRepository = validationJobRepository; this.validationIssueRepository = validationIssueRepository; this.validationResultRepository = validationResultRepository; this.latch = latch; @@ -41,13 +46,18 @@ public class OaiPmhRoute extends RouteBuilder { String date = new Date().toString(); - from(oaiEndpoint).routeId(date) + from(oaiEndpoint) + .routeId(date) .split(xpath("//*[local-name()='record']")) .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository, maxNumberOfRecords)) .choice().when(header("MyHeader").isEqualTo("stop")) + .to("direct:saveToDatabase") .to("controlbus:route?routeId="+date+"&action=stop") .endChoice(); + from("direct:saveToDatabase") + .process(new DataBaseProcessor(validationJob, validationJobRepository, validationResultRepository)); + /*from("timer://myTimer?fixedRate=true&period=60000") .routeId("OAIProcessingRoute") .setHeader("MyHeader", constant("stop"))