Added separate processor to save in db + refined the APIs for the job and the error/warnings + added repository method for the status of a job

src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java

@@ -25,37 +25,37 @@ public class ReportController {
             this.validationJobRepository = validationJobRepository;
             this.validationResultRepository = validationResultRepository;
             this.validationIssueRepository = validationIssueRepository;
-        }
+    }
+
+    @RequestMapping(value={"getJobResult"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
+    public ValidationJob getJobResults(@RequestParam(name = "jobId") int jobId){
+        Optional<ValidationJob> validationJob = validationJobRepository.findById(jobId);
+        return (ValidationJob) validationJob.orElse(null);
+    }
 
-        @RequestMapping(value={"getJobResult"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
-        public ValidationJob getJobResults(@RequestParam(name = "jobId") int jobId){
-            Optional<ValidationJob> validationJob = validationJobRepository.findById(jobId);
-            return (ValidationJob) validationJob.orElse(null);
-        }
     @RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
     public List<SummaryResult> getSummaryJobResults(@RequestParam(name = "jobId") int jobId){
         return validationResultRepository.getSummaryResult(jobId);
     }
 
+    @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
+    public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
+        List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrlsByRuleName(jobId, ruleName);
+        return extractRecordsGroupedByRule(resultList);
+    }
+
+    @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
+    public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
+        List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrlsByRuleName(jobId, ruleName);
+        return extractRecordsGroupedByRule(resultList);
+    }
+
     @RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
     public List<ValidationIssue> getRecordsByRule(@RequestParam(name = "jobId") int jobId,
                                                   @RequestParam(name = "ruleName") String ruleName){
         return  validationIssueRepository.getErrors(jobId, ruleName);
     }
 
-    @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
-    public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId){
-        List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId);
-        return extractRecordsGroupedByRule(resultList);
-    }
-
-    @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
-    public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId,
-                                                       @RequestParam(name = "ruleName") String ruleName){
-        List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId);
-        return extractRecordsGroupedByRule(resultList);
-    }
-
     private Map<String, List<String>> extractRecordsGroupedByRule(List<Object[]> resultList) {
         Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
 

src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java

@@ -129,13 +129,13 @@ public class ValidationController {
                     System.out.println(record++);
                 }
             }
-            validationJob.status = "COMPLETED";
+            validationJob.progress = "COMPLETED";
 
         }
         catch (Exception e) {
             log.error("Validation job stopped unexpectedly." + e.getMessage());
             System.out.println("ERROR " + e.getMessage());
-            validationJob.status = "STOPPED";
+            validationJob.progress = "STOPPED";
 
         } finally {
             validationJob.endDate = new Date();
@@ -143,6 +143,7 @@ public class ValidationController {
             validationJob.recordsTested = record;
             validationJob.score = resultSum / record;
             //TODO uncomment
+            log.info("Saving validation job " + validationJob.recordsTested);
             validationJobRepository.save(validationJob);
         }
 
@@ -197,11 +198,9 @@ public class ValidationController {
 
         ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords);
         validationJob.guidelines = profile.name();
-        validationJob.status = "IN_PROGRESS";
         validationJobRepository.save(validationJob);
         log.info("Initial validation job id "+ validationJob.id);
 
-
         int record = 0;
         double resultSum = 0;
 
@@ -211,24 +210,21 @@ public class ValidationController {
 
             RouteBuilder oaiPmhRouteBuilder =
                     new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
-                            profile, validationJob, numberOfRecords, validationIssueRepository, validationResultRepository);
+                            profile, validationJob, numberOfRecords, validationJobRepository,
+                            validationIssueRepository, validationResultRepository);
             camelContext.addRoutes(oaiPmhRouteBuilder);
-            validationJob.status = "COMPLETED";
+            validationJob.progress = "COMPLETED";
 
         }
         catch (Exception e) {
             log.error("Validation job stopped unexpectedly." +  e.getMessage());
             System.out.println("ERROR " + e.getMessage());
-            validationJob.status = "STOPPED";
+            validationJob.progress = "STOPPED";
-
-        } finally {
             validationJob.endDate = new Date();
-            log.info("Final validation job "+ validationJob.id);
+            validationJob.score = resultSum / validationJob.recordsTested;
-            validationJob.recordsTested = record;
+            validationJob.status = validationResultRepository.getStatus(validationJob.id);
-            validationJob.score = resultSum / record;
             validationJobRepository.save(validationJob);
         }
-
     }
 
 
@@ -425,12 +421,13 @@ public class ValidationController {
                 }
                 record++;
             }
-            validationJob.status = "COMPLETED";
+
+            validationJob.progress = "COMPLETED";
 
         }
         catch (Exception e) {
             log.error("Validation job stopped unexpectedly." + e.getMessage());
-            validationJob.status = "STOPPED";
+            validationJob.progress = "STOPPED";
 
         } finally {
             validationJob.endDate = new Date();

src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java

@@ -8,7 +8,7 @@ import java.util.Date;
 public class ValidationJob {
     @GeneratedValue(strategy = GenerationType.IDENTITY)
     @Id
-    public int id; //not in use for now
+    public int id;
 
     @Column(name = "base_url")
     public String baseUrl;
@@ -24,8 +24,10 @@ public class ValidationJob {
 
     @Column(name = "records_tested")
     public int recordsTested;
-    @Column(name="duration")
+    @Column(name="progress")
-    public String status; //stopped, completed, in progress
+    public String progress; //stopped, completed, in progress
+    @Column(name="status")
+    public String status; //success, failure
     @Column(name="score")
     public double score;
 
@@ -42,7 +44,7 @@ public class ValidationJob {
 
     @Override
     public String toString() {
-        return "ValidationJob {" +
+        return "ValidationJob{" +
                 "id=" + id +
                 ", baseUrl='" + baseUrl + '\'' +
                 ", numberOfRecords=" + numberOfRecords +
@@ -50,8 +52,12 @@ public class ValidationJob {
                 ", startDate=" + startDate +
                 ", endDate=" + endDate +
                 ", recordsTested=" + recordsTested +
+                ", progress='" + progress + '\'' +
                 ", status='" + status + '\'' +
                 ", score=" + score +
                 '}';
     }
+
+
+
 }

src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java

@@ -0,0 +1,32 @@
+package eu.dnetlib.validatorapi.processors;
+
+import eu.dnetlib.validatorapi.entities.ValidationJob;
+import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+
+import java.util.Date;
+
+public class DataBaseProcessor implements Processor {
+    private final ValidationJob validationJob;
+    private final ValidationJobRepository validationJobRepository;
+    private final ValidationResultRepository validationResultRepository;
+
+    public DataBaseProcessor(ValidationJob validationJob, final ValidationJobRepository validationJobRepository,
+                             final ValidationResultRepository validationResultRepository){
+        this.validationJob = validationJob;
+        this.validationJobRepository = validationJobRepository;
+        this.validationResultRepository = validationResultRepository;
+
+    }
+
+    @Override
+    public void process(Exchange exchange) throws Exception {
+        validationJob.progress = "COMPLETED";
+        validationJob.endDate = new Date();
+        validationJob.status = validationResultRepository.getStatus(validationJob.id);
+        System.out.println(validationJob.status);
+        validationJobRepository.save(validationJob);
+    }
+}

src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java

@@ -1,4 +0,0 @@
-package eu.dnetlib.validatorapi.processors;
-
-public class StopRouteProcessor {
-}

src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java

@@ -22,14 +22,15 @@ import java.util.Map;
 
 public class XmlProcessor implements Processor {
     private final AbstractOpenAireProfile profile;
-    private final ValidationJob validationJob;
+    private ValidationJob validationJob;
     private final ValidationIssueRepository validationIssueRepository;
     private final ValidationResultRepository validationResultRepository;
     private final long maxNumberOfRecords;
-    private long processedRecords;
+    private int processedRecords;
+    private int resultSum;
 
 
-    public XmlProcessor(final AbstractOpenAireProfile profile, final ValidationJob validationJob,
+    public XmlProcessor(final AbstractOpenAireProfile profile, ValidationJob validationJob,
                         final ValidationIssueRepository validationIssueRepository,
                         final ValidationResultRepository validationResultRepository,
                         final long maxNumberOfRecords){
@@ -64,13 +65,16 @@ public class XmlProcessor implements Processor {
                 constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
                         ruleName, profile, engineResult);
                 validationResultRepository.save(validationRuleResult);
+                resultSum += engineResult.score();
                 saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR");
                 saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING");
             }
         }
 
         processedRecords++;
-        exchange.setProperty("totalRecords", processedRecords);
+        validationJob.recordsTested = processedRecords;
+        validationJob.score = resultSum;
+        System.out.println("\n\n\nvalidation job: " + processedRecords);
 
         if (processedRecords > maxNumberOfRecords) {
             exchange.getIn().setHeader("MyHeader", "stop");

src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java

@@ -26,14 +26,26 @@ public interface ValidationIssueRepository extends JpaRepository<ValidationIssue
 
     @Query(value = "SELECT vi.issueText, vi.recordUrl " +
             "FROM ValidationIssue vi " +
-            "WHERE (vi.issueType = 'ERROR'  AND vi.validationJobId =:id)" +
+            "WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id AND vi.ruleName=:ruleName)" +
             "GROUP BY vi.issueText, vi.recordUrl")
-    List<Object[]> getAllErrorsRecordUrls(@Param("id") int validationJobId);
+    List<Object[]> getAllErrorsRecordUrlsByRuleName(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
 
     @Query(value = "SELECT vi.issueText, vi.recordUrl " +
             "FROM ValidationIssue vi " +
-            "WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id)" +
+            "WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id AND vi.ruleName=:ruleName)" +
             "GROUP BY vi.issueText, vi.recordUrl")
-    List<Object[]> getAllWarningsRecordUrls(@Param("id") int validationJobId);
+    List<Object[]> getAllWarningsRecordUrlsByRuleName(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
+
+    @Query(value = "SELECT vi.ruleName, vi.issueText, vi.recordUrl " +
+            "FROM ValidationIssue vi " +
+            "WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" +
+            "GROUP BY vi.ruleName, vi.issueText, vi.recordUrl")
+    List<Object[]> getAllErrors(@Param("id") int validationJobId);
+
+    @Query(value = "SELECT vi.ruleName, vi.issueText, vi.recordUrl " +
+            "FROM ValidationIssue vi " +
+            "WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id)" +
+            "GROUP BY vi.ruleName, vi.issueText, vi.recordUrl")
+    List<Object[]> getAllWarnings(@Param("id") int validationJobId);
 
 }

src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java

@@ -19,4 +19,10 @@ public interface ValidationResultRepository extends JpaRepository<ValidationRule
             "FROM ValidationRuleResult sr " + "WHERE sr.validationJobId = :id " +
             "GROUP BY sr.ruleName, sr.ruleWeight")
     List<SummaryResult> getSummaryResult(@Param("id") int validationJobId);
+
+    @Query("SELECT CASE WHEN COUNT(vr) > 0 THEN 'FAILURE' ELSE 'SUCCESS' END " +
+            "FROM ValidationRuleResult vr " +
+            "WHERE vr.validationJobId = :id AND vr.status = 'FAILURE'")
+    public String getStatus(@Param("id") int validationJobId);
+
 }

src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java

@@ -2,8 +2,10 @@ package eu.dnetlib.validatorapi.routes;
 
 import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
 import eu.dnetlib.validatorapi.entities.ValidationJob;
+import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
 import eu.dnetlib.validatorapi.processors.XmlProcessor;
 import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
 import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
@@ -19,18 +21,21 @@ public class OaiPmhRoute extends RouteBuilder {
     private AbstractOpenAireProfile profile;
     private long maxNumberOfRecords = 0;
     private ValidationJob validationJob;
+    private final ValidationJobRepository validationJobRepository;
     private final ValidationIssueRepository validationIssueRepository;
     private final ValidationResultRepository validationResultRepository;
      CountDownLatch latch;
 
 
     public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob,
-                       long maxNumberOfRecords, final ValidationIssueRepository validationIssueRepository,
+                       long maxNumberOfRecords, final ValidationJobRepository validationJobRepository,
+                       final ValidationIssueRepository validationIssueRepository,
                        final ValidationResultRepository validationResultRepository) {
         this.oaiEndpoint = oaiEndpoint;
         this.validationJob = validationJob;
         this.profile = profile;
         this.maxNumberOfRecords = maxNumberOfRecords;
+        this.validationJobRepository = validationJobRepository;
         this.validationIssueRepository = validationIssueRepository;
         this.validationResultRepository = validationResultRepository;
         this.latch = latch;
@@ -41,13 +46,18 @@ public class OaiPmhRoute extends RouteBuilder {
 
         String date = new Date().toString();
 
-        from(oaiEndpoint).routeId(date)
+        from(oaiEndpoint)
+                .routeId(date)
                 .split(xpath("//*[local-name()='record']"))
                 .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository, maxNumberOfRecords))
                 .choice().when(header("MyHeader").isEqualTo("stop"))
+                     .to("direct:saveToDatabase")
                      .to("controlbus:route?routeId="+date+"&action=stop")
                 .endChoice();
 
+        from("direct:saveToDatabase")
+                .process(new DataBaseProcessor(validationJob, validationJobRepository, validationResultRepository));
+
         /*from("timer://myTimer?fixedRate=true&period=60000")
                 .routeId("OAIProcessingRoute")
                 .setHeader("MyHeader", constant("stop"))