added summary table and cleaning processes for all validation cases

src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java

@@ -3,6 +3,7 @@ package eu.dnetlib.validatorapi.controllers;
 import eu.dnetlib.validatorapi.entities.IssueDescriptor;
 import eu.dnetlib.validatorapi.entities.SummaryResult;
 import eu.dnetlib.validatorapi.entities.ValidationJob;
+import eu.dnetlib.validatorapi.repositories.SummaryValidationJobRepository;
 import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
 import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
 import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
@@ -20,13 +21,16 @@ public class ReportController {
     private final ValidationJobRepository validationJobRepository;
     private final ValidationResultRepository validationResultRepository;
     private final ValidationIssueRepository validationIssueRepository;
+    private final SummaryValidationJobRepository summaryValidationJobRepository;
 
     public ReportController(ValidationJobRepository validationJobRepository,
                             ValidationResultRepository validationResultRepository,
-                            ValidationIssueRepository validationIssueRepository) {
+                            ValidationIssueRepository validationIssueRepository,
+                            SummaryValidationJobRepository summaryValidationJobRepository) {
             this.validationJobRepository = validationJobRepository;
             this.validationResultRepository = validationResultRepository;
             this.validationIssueRepository = validationIssueRepository;
+            this.summaryValidationJobRepository = summaryValidationJobRepository;
     }
 
     @RequestMapping(value={"getJobResult"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
@@ -44,6 +48,38 @@ public class ReportController {
         return validationResultRepository.getFullSummaryResult(jobId);
     }
 
+    @RequestMapping(value = {"saveResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
+    public void saveSummaryJobResults(@RequestParam(name = "jobId") int jobId,
+                                                    @RequestParam(name= "guidelines", required = false) Optional<String> guidelines) {
+        List<SummaryResult> summaryResultList;
+        if (guidelines.isPresent() && guidelines.get().isEmpty())
+            summaryResultList = validationResultRepository.getFullSummaryResult(jobId, guidelines.get());
+        else
+            summaryResultList = validationResultRepository.getFullSummaryResult(jobId);
+
+        for(SummaryResult sr:summaryResultList){
+            System.out.println("Saving " + sr.getRuleName() + ", " + sr.getValidationJobId());
+            sr.setValidationJobId(jobId);
+            summaryValidationJobRepository.save(sr);
+        }
+    }
+
+    @RequestMapping(value = {"getSummaryFromDB"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
+    public List<SummaryResult> getSummaryFromDB(@RequestParam(name = "jobId") int jobId,
+                                      @RequestParam(name= "guidelines", required = false) Optional<String> guidelines) {
+        List<SummaryResult> srlist =  summaryValidationJobRepository.findByValidationJobIdOrderByRuleName(jobId);
+
+       /* SummaryResult sr = new SummaryResult();
+        sr.setValidationJobId(jobId);
+        sr.setRule_name("Access Rights");
+        System.out.println("\n>>" + summaryValidationJobRepository.findAll());*/
+       /* for (SummaryResult sr:srlist) {
+            System.out.println(sr);
+        }*/
+        return srlist;
+
+    }
+
     @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
     private List<IssueDescriptor> getWarningDescriptors(@RequestParam(name = "jobId") int jobId,
                                                       @RequestParam(name = "ruleName") String ruleName){

src/main/java/eu/dnetlib/validatorapi/controllers/SummaryResultId.java

@@ -0,0 +1,33 @@
+package eu.dnetlib.validatorapi.controllers;
+
+import java.io.Serializable;
+
+public class SummaryResultId implements Serializable {
+    public String ruleName;
+    public int validationJobId;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        SummaryResultId that = (SummaryResultId) o;
+
+        System.out.println("\n\n\n\n");
+        System.out.println("Validation job id " + validationJobId);
+        System.out.println("rule_name " + ruleName);
+        System.out.println("\n\n\n\n");
+
+
+        if (validationJobId != that.validationJobId) return false;
+        return ruleName != null ? ruleName.equals(that.ruleName) : that.ruleName == null;
+    }
+
+    // Implement hashCode method
+    @Override
+    public int hashCode() {
+        int result = ruleName != null ? ruleName.hashCode() : 0;
+        result = 31 * result + validationJobId;
+        return result;
+    }
+}

src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java

@@ -7,12 +7,14 @@ import eu.dnetlib.validatorapi.entities.RuleInfo;
 import eu.dnetlib.validatorapi.entities.ValidationJob;
 import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
 import eu.dnetlib.validatorapi.exceptions.ValidationException;
+import eu.dnetlib.validatorapi.repositories.SummaryValidationJobRepository;
 import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
 import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
 import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
 import eu.dnetlib.validatorapi.routes.FairOaiPmhRoute2;
 import eu.dnetlib.validatorapi.routes.SimpleOaiPmhRoute;
 import org.apache.camel.CamelContext;
+import org.apache.camel.Exchange;
 import org.apache.camel.ProducerTemplate;
 import org.apache.camel.builder.RouteBuilder;
 import org.apache.http.HttpStatus;
@@ -55,16 +57,20 @@ public class ValidationController {
     private final ValidationResultRepository validationResultRepository;
     private final ValidationIssueRepository validationIssueRepository;
 
+    private final SummaryValidationJobRepository summaryValidationJobRepository;
+
     @Autowired
     CamelContext camelContext;
 
     @Autowired
     public ValidationController(ValidationJobRepository validationJobRepository,
                                 ValidationResultRepository validationResultRepository,
-                                ValidationIssueRepository validationIssueRepository) {
+                                ValidationIssueRepository validationIssueRepository,
+                                SummaryValidationJobRepository summaryValidationJobRepository) {
         this.validationJobRepository = validationJobRepository;
         this.validationResultRepository = validationResultRepository;
         this.validationIssueRepository = validationIssueRepository;
+        this.summaryValidationJobRepository = summaryValidationJobRepository;
     }
 
     @RequestMapping(value = {"/realValidator-old"}, method = RequestMethod.GET)
@@ -145,7 +151,6 @@ public class ValidationController {
             //System.out.println("Final validation job "+ validationJob.hashCode());
             validationJob.recordsTested = record;
             validationJob.score = resultSum / record;
-            //TODO uncomment
             log.info("Saving validation job " + validationJob.recordsTested);
             validationJobRepository.save(validationJob);
         }
@@ -219,11 +224,15 @@ public class ValidationController {
 
             if (fairProfile == null) {
                 oaiPmhRouteBuilder =
-                        new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
+                        new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString(),
+                                validationJobRepository, validationResultRepository, summaryValidationJobRepository);
 
+                     //   new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
             } else {
                 oaiPmhRouteBuilder =
-                        new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
+                       // new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
+                        new FairOaiPmhRoute2(oaiEndpoint, profile, fairProfile, validationJob, numberOfRecords, uuid.toString(),
+                                validationJobRepository, validationResultRepository, summaryValidationJobRepository);
 
             }
 
@@ -252,14 +261,20 @@ public class ValidationController {
         camelContext.addRoutes(oaiSetListRoute);
     }*/
 
-        @RequestMapping(value = {"/getSets"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
-    public String getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
+    @RequestMapping(value = {"/getSets"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
+    public ResponseEntity<String> getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
                 ) throws Exception {
 
         ProducerTemplate producerTemplate = camelContext.createProducerTemplate();
-        String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);
+        /*String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);*/
+        Exchange message = producerTemplate.request("direct:getResponse", exchange -> {
+            exchange.getMessage().setHeader("endpoint", "oaipmh://" + baseURL + "?verb=ListSets");
+        });
 
-        return response;
+        return ResponseEntity
+                //TODO: maybe something better if header is not set (null and mot 200)
+                .status(message.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, 200, Integer.class))
+                .body(message.getIn().getBody(String.class));
 
 /*git git
         System.out.println(camelContext.getRoutes());
@@ -417,6 +432,9 @@ public class ValidationController {
             return new FAIR_Data_GuidelinesProfile();
 
         } else if (guidelinesProfileName.equals("OpenAIRE Guidelines for Literature Repositories Profile v4")) {
+            System.out.println("\n\n\nRETURNING FAIR LITERATURE\n\n\n");
+            System.out.println("\n\n\n" + new FAIR_Literature_GuidelinesV4Profile().name() + "\n\n\n");
+
             return new FAIR_Literature_GuidelinesV4Profile();
         }
 

src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java

@@ -1,66 +1,72 @@
 package eu.dnetlib.validatorapi.entities;
 
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import javax.persistence.IdClass;
+import eu.dnetlib.validatorapi.controllers.SummaryResultId;
+
+import javax.persistence.*;
 import java.io.Serializable;
 
 @Entity
-@IdClass(SummaryResult.class)
+@Table(name="summary_result")
+@IdClass(SummaryResultId.class)
 public class SummaryResult implements Serializable {
     @Id
     @Column(name="rule_name")
-    String rule_name;
+    public String ruleName;
 
     @Id
     @Column(name="validation_job_id")
-    int validationJobId;
+    public int validationJobId;
 
     @Column(name="rule_weight")
-    int rule_weight;
+    public int rule_weight;
 
     @Column(name="requirement_level")
-    String requirement_level;
+    public String requirement_level;
 
     @Column(name="description")
-    String description;
+    public String description;
 
     @Column(name="fair_principles")
-    String fair_principles;
+    public String fair_principles;
 
     @Column(name="link")
-    String link;
+    public String link;
 
     @Column(name="guidelines")
-    String guidelines;
+    public String guidelines;
 
     @Column(name = "internal_error")
-    String internal_error;
+    public String internal_error;
 
     @Column(name="rule_status")
-    String rule_status;
+    public String rule_status;
 
     @Column(name = "passed_records")
-    long passed_records;
+    public long passed_records;
 
     @Column(name="failed_records")
-    long failed_records;
+    public long failed_records;
 
     @Column(name="has_errors")
-    boolean has_errors;
+    public boolean has_errors;
 
     @Column(name="has_warnings")
-    boolean has_warnings;
+    public boolean has_warnings;
 
+    public SummaryResult(){}
+
+    public SummaryResult(int validationJobId, String rule_name) {
+        this.validationJobId = validationJobId;
+        this.ruleName = rule_name;
+    }
     public SummaryResult(String rule_name, long passed_records, long failed_records) {
-        this.rule_name = rule_name;
+        this.ruleName = rule_name;
         this.passed_records = passed_records;
         this.failed_records = failed_records;
     }
 
     public SummaryResult(String rule_name, int rule_weight, long passed_records, long failed_records) {
-        this.rule_name = rule_name;
+        this.ruleName = rule_name;
         this.rule_weight = rule_weight;
         this.passed_records = passed_records;
         this.failed_records = failed_records;
@@ -70,7 +76,7 @@ public class SummaryResult implements Serializable {
     public SummaryResult(String rule_name, int rule_weight, String requirement_level, String description, String fair_principles,
                          String link, String guidelines, String internal_error, long passed_records, long failed_records,
                          String rule_status, boolean has_errors, boolean has_warnings) {
-        this.rule_name = rule_name;
+        this.ruleName = rule_name;
         this.rule_weight = rule_weight;
         this.requirement_level = requirement_level;
         this.description = description;
@@ -87,12 +93,20 @@ public class SummaryResult implements Serializable {
 
 
 
-    public String getRule_name() {
-        return rule_name;
+    public String getRuleName() {
+        return ruleName;
     }
 
-    public void setRule_name(String rule_name) {
-        this.rule_name = rule_name;
+    public void setRuleName(String ruleName) {
+        this.ruleName = ruleName;
+    }
+
+    public int getValidationJobId() {
+        return validationJobId;
+    }
+
+    public void setValidationJobId(int validationJobId) {
+        this.validationJobId = validationJobId;
     }
 
     public int getRule_weight() {
@@ -194,12 +208,11 @@ public class SummaryResult implements Serializable {
     @Override
     public String toString() {
         return "SummaryResult{" +
-                "rule_name='" + rule_name + '\'' +
+                "rule_name='" + ruleName + '\'' +
                 ", rule_weight=" + rule_weight +
                 ", passed_records=" + passed_records +
                 ", failed_records=" + failed_records +
                 '}';
     }
 
-
 }

src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java

@@ -36,7 +36,7 @@ public class ValidationIssue implements Serializable {
         this.issueType = issueType;
     }
 
-    public int getValidationJobId() {
+    public long getValidationJobId() {
         return validationJobId;
     }
 

src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java

@@ -22,6 +22,7 @@ public class DataBaseProcessor implements Processor {
         validationJob.score = (Integer) exchange.getIn().getHeader("score");
         validationJob.progress = "COMPLETED";
         validationJob.endDate = new Date();
+
         exchange.getIn().setBody(validationJob, ValidationJob.class);
     }
 }

src/main/java/eu/dnetlib/validatorapi/processors/SetsProcessor.java

@@ -0,0 +1,90 @@
+package eu.dnetlib.validatorapi.processors;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.support.DefaultMessage;
+import org.apache.http.HttpStatus;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.json.JSONObject;
+import org.json.XML;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+public class SetsProcessor  implements Processor {
+
+    private final Logger log = LogManager.getLogger(this.getClass());
+
+    @Override
+    public void process(Exchange exchange) throws Exception {
+        List<DefaultMessage> messages = exchange.getIn().getBody(ArrayList.class);
+
+        //If the body is null a timeout has occurred.
+        if (exchange.getIn().getBody() == null)
+            throw new TimeoutException();
+
+        List<String> ListSetsXml = exchange.getIn().getBody(ArrayList.class);
+        List<String> desiredJsonParts = new ArrayList<>();
+        String xmlSets = "";
+
+        // Extract the desired <set> elements from each XML string
+        for (String xml : ListSetsXml) {
+            List<String> extractedSets = extractSets(xml);
+            for (String setXml : extractedSets)
+                xmlSets += setXml;
+        }
+
+        JSONObject jsonObject = XML.toJSONObject(xmlSets);
+        String jsonString = jsonObject.toString();
+        exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, HttpStatus.SC_OK);
+        exchange.getIn().setBody(jsonString);
+    }
+
+    private List<String> extractSets(String xml) {
+        List<String> setXmls = new ArrayList<>();
+        try {
+            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+            DocumentBuilder builder = factory.newDocumentBuilder();
+            InputSource inputSource = new InputSource(new StringReader(xml));
+            Document document = builder.parse(inputSource);
+
+            NodeList setElements = document.getElementsByTagName("set");
+            for (int i = 0; i < setElements.getLength(); i++) {
+                Node setElement = setElements.item(i);
+                String setXml = nodeToString(setElement);
+                setXmls.add(setXml);
+            }
+        } catch (Exception e) {
+            // TODO: consider if Ineed to handle any exceptions that occur during
+            //  XML parsing or processing and not only log them
+           log.error("Could not parse the xml ", e);
+        }
+
+        return setXmls;
+    }
+
+    private String nodeToString(Node node) throws TransformerException {
+        StringWriter sw = new StringWriter();
+        TransformerFactory tf = TransformerFactory.newInstance();
+        Transformer transformer = tf.newTransformer();
+        transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
+        transformer.transform(new DOMSource(node), new StreamResult(sw));
+        return sw.toString();
+    }
+}

src/main/java/eu/dnetlib/validatorapi/processors/SumUpNCleanProcessor.java

@@ -0,0 +1,63 @@
+package eu.dnetlib.validatorapi.processors;
+
+import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
+import eu.dnetlib.validatorapi.entities.SummaryResult;
+import eu.dnetlib.validatorapi.entities.ValidationJob;
+import eu.dnetlib.validatorapi.repositories.SummaryValidationJobRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+
+import java.util.Date;
+import java.util.List;
+
+public class SumUpNCleanProcessor implements Processor {
+
+    private final ValidationJobRepository validationJobRepository;
+    private final ValidationResultRepository validationResultRepository;
+    private final SummaryValidationJobRepository summaryValidationJobRepository;
+    private final AbstractOpenAireProfile profile;
+
+    public SumUpNCleanProcessor(ValidationJobRepository validationJobRepository,
+                                ValidationResultRepository validationResultRepository,
+                                SummaryValidationJobRepository summaryValidationJobRepository,
+                                AbstractOpenAireProfile profile) {
+        this.validationJobRepository = validationJobRepository;
+        this.validationResultRepository = validationResultRepository;
+        this.summaryValidationJobRepository = summaryValidationJobRepository;
+        this.profile = profile;
+    }
+
+    @Override
+    public void process(Exchange exchange) throws Exception {
+
+        Thread stop=null;
+
+        ValidationJob validationJob = new ValidationJob();
+        validationJob.id = (Integer)exchange.getIn().getHeader("validationId");
+        validationJob.baseUrl = (String)(exchange.getIn().getHeader("baseUrl"));
+        validationJob.set = (String)(exchange.getIn().getHeader("set"));
+        validationJob.startDate = (Date) exchange.getIn().getHeader("startDate");
+        validationJob.guidelines = (String) exchange.getIn().getHeader("guidelines");
+        validationJob.status = (String)exchange.getIn().getHeader("status");
+        validationJob.recordsTested = (Integer) exchange.getIn().getHeader("recordsTested");
+        validationJob.numberOfRecords = Math.toIntExact((Long) exchange.getIn().getHeader("maxNumberOfRecords"));
+        validationJob.score = (Integer) exchange.getIn().getHeader("score");
+        validationJob.progress = "COMPLETED";
+        validationJob.endDate = new Date();
+
+        List<SummaryResult> summaryResults  = validationResultRepository.getFullSummaryResult
+                ((Integer) exchange.getIn().getHeader("validationId"));
+
+        for(SummaryResult sr: summaryResults) {
+            sr.validationJobId = validationJob.id;
+            summaryValidationJobRepository.save(sr);
+        }
+
+        validationResultRepository.customDeleteByGuidelines(validationJob.id, profile.name());
+        validationJobRepository.save(validationJob);
+
+    }
+
+}

src/main/java/eu/dnetlib/validatorapi/processors/SummaryValidationJobProcessor.java

@@ -0,0 +1,41 @@
+package eu.dnetlib.validatorapi.processors;
+
+import eu.dnetlib.validatorapi.entities.SummaryResult;
+import eu.dnetlib.validatorapi.repositories.SummaryValidationJobRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+
+import java.util.List;
+
+public class SummaryValidationJobProcessor implements Processor {
+
+    private final ValidationResultRepository validationResultRepository;
+    private final SummaryValidationJobRepository summaryValidationJobRepository;
+    private final String routeId;
+
+    public SummaryValidationJobProcessor(ValidationResultRepository validationResultRepository,
+                                  SummaryValidationJobRepository summaryValidationJobRepository,
+                                         String routeId){
+        this.validationResultRepository = validationResultRepository;
+        this.summaryValidationJobRepository = summaryValidationJobRepository;
+        this.routeId = routeId;
+    }
+
+    @Override
+    public void process(Exchange exchange) throws Exception {
+        int validation_job_id = (Integer) exchange.getIn().getHeader("validationId");
+        List<SummaryResult> summaryResults  = validationResultRepository.getFullSummaryResult
+                ((Integer) exchange.getIn().getHeader("validationId"));
+
+        for(SummaryResult sr: summaryResults) {
+            sr.validationJobId = validation_job_id;
+            summaryValidationJobRepository.save(sr);
+        }
+
+        validationResultRepository.customDelete(validation_job_id);
+
+        exchange.getIn().setHeader("CamelControlBusAction", "stop");
+        exchange.getIn().setHeader("CamelControlBusRouteId", routeId);
+    }
+}

src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java

@@ -48,10 +48,10 @@ public class XmlProcessor implements Processor {
         final DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
         Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml)));
         String recordUrl = extractRecordUrl(doc, "identifier");
-        System.out.println("Identifier: " + recordUrl);
         List<ValidationRuleResult> validationRuleResults = new ArrayList<>();
         List<ValidationIssue> validationIssues = new ArrayList<>();
 
+        System.out.println("\n\n\nProcess with guideline " + profile.name());
         if (profile != null) {
             XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
             Map<String, Guideline.Result> results = validationResult.results();
@@ -98,17 +98,26 @@ public class XmlProcessor implements Processor {
 
        // System.out.println("Validation Issues: " + validationIssues);
 
+        exchange.getIn().setHeader("MyHeader", "SKATA");
+
         if (processedRecords > maxNumberOfRecords) {
             exchange.getIn().setHeader("MyHeader", "stop");
             exchange.getIn().setHeader("validationId", validationJob.id);
             exchange.getIn().setHeader("baseUrl", validationJob.baseUrl);
             exchange.getIn().setHeader("set", validationJob.set);
             exchange.getIn().setHeader("startDate", validationJob.startDate);
+            System.out.println("\n\n\n" + validationJob.id +": set guidelines "+ validationJob.guidelines + "\n\n\n");
             exchange.getIn().setHeader("guidelines", validationJob.guidelines);
             exchange.getIn().setHeader("status", status);
             exchange.getIn().setHeader("recordsTested", processedRecords-1);
             exchange.getIn().setHeader("maxNumberOfRecords", maxNumberOfRecords);
             exchange.getIn().setHeader("score", scoreSum/processedRecords);
+
+
+            System.out.println("\n\n\n\n\n PLACE STOP HEADER \n\n\n\n\n\n");
+
+
+            //exchange.setRouteStop(true);
         }
     }
 

src/main/java/eu/dnetlib/validatorapi/repositories/SummaryValidationJobRepository.java

@@ -4,7 +4,10 @@ import eu.dnetlib.validatorapi.entities.SummaryResult;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
-@Repository
-public interface SummaryValidationJobRepository extends JpaRepository<SummaryResult, Long> {
+import java.util.List;
 
+@Repository
+public interface SummaryValidationJobRepository extends JpaRepository<SummaryResult, Integer> {
+
+    List<SummaryResult> findByValidationJobIdOrderByRuleName(int validationJobId);
 }

src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java

@@ -3,14 +3,16 @@ package eu.dnetlib.validatorapi.repositories;
 import eu.dnetlib.validatorapi.entities.SummaryResult;
 import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
 
 @Repository
-public interface ValidationResultRepository extends JpaRepository<ValidationRuleResult, Long> {
+public interface ValidationResultRepository extends JpaRepository<ValidationRuleResult, Integer> {
 
     @Query(value =
             "SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," +
@@ -39,7 +41,8 @@ public interface ValidationResultRepository extends JpaRepository<ValidationRule
             "CASE WHEN COUNT(CASE WHEN has_warnings IS TRUE THEN 1 END) > 0 THEN TRUE ELSE FALSE END AS hasWarnings)" +
             "FROM eu.dnetlib.validatorapi.entities.ValidationRuleResult sr " +
             "WHERE sr.validationJobId = :id " +
-            "GROUP BY sr.ruleName, sr.ruleWeight, sr.description, sr.fair_principles, sr.link, sr.requirement_level, sr.guidelines,  sr.internalError, sr.validationJobId")
+            "GROUP BY sr.ruleName, sr.ruleWeight, sr.description, sr.fair_principles, sr.link, sr.requirement_level, sr.guidelines,  sr.internalError, sr.validationJobId " +
+            "ORDER BY sr.ruleName")
     List<SummaryResult> getFullSummaryResult(@Param("id") int validationJobId);
 
     @Query(value =
@@ -73,4 +76,13 @@ public interface ValidationResultRepository extends JpaRepository<ValidationRule
     List<SummaryResult> getFullSummaryResult(@Param("id") int validationJobId, @Param("guidelines") String guidelines);
 
 
+    @Modifying
+    @Transactional
+    @Query("DELETE FROM ValidationRuleResult vr WHERE vr.validationJobId = :validationJobId")
+    void customDelete(int validationJobId);
+
+    @Modifying
+    @Transactional
+    @Query("DELETE FROM ValidationRuleResult vr WHERE vr.validationJobId = :validationJobId and vr.guidelines =:guidelines")
+    void customDeleteByGuidelines(int validationJobId, String guidelines);
 }

src/main/java/eu/dnetlib/validatorapi/routes/FairOaiPmhRoute2.java

@@ -1,14 +1,16 @@
 package eu.dnetlib.validatorapi.routes;
 
 import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
-import eu.dnetlib.validator2.validation.guideline.openaire.FAIR_Data_GuidelinesProfile;
 import eu.dnetlib.validatorapi.entities.ValidationIssue;
 import eu.dnetlib.validatorapi.entities.ValidationJob;
 import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
-import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
 import eu.dnetlib.validatorapi.processors.ErrorProcessor;
 import eu.dnetlib.validatorapi.processors.ExceptionProcessor;
+import eu.dnetlib.validatorapi.processors.SumUpNCleanProcessor;
 import eu.dnetlib.validatorapi.processors.XmlProcessor;
+import eu.dnetlib.validatorapi.repositories.SummaryValidationJobRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
 import org.apache.camel.TypeConversionException;
@@ -19,18 +21,25 @@ import org.apache.http.client.ClientProtocolException;
 import java.net.UnknownHostException;
 import java.util.Date;
 import java.util.UUID;
+import java.util.concurrent.RejectedExecutionException;
 
 public class FairOaiPmhRoute2 extends RouteBuilder {
 
     private String oaiEndpoint;
 
-    private AbstractOpenAireProfile profile;
+    private final AbstractOpenAireProfile profile;
+    private final AbstractOpenAireProfile fairProfile;
     private long maxNumberOfRecords = 0;
-    private ValidationJob validationJob;
-    String routeid;
-    String routeid2;
+    private final ValidationJob validationJob;
+    private String routeid;
+    private String routeid2;
+
+    private ValidationJobRepository validationJobRepository;
+    private ValidationResultRepository validationResultRepository;
+    private SummaryValidationJobRepository summaryValidationJobRepository;
 
 
+/*
     public FairOaiPmhRoute2(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob,
                             long maxNumberOfRecords, String routeid) {
         this.oaiEndpoint = oaiEndpoint;
@@ -40,6 +49,24 @@ public class FairOaiPmhRoute2 extends RouteBuilder {
         this.routeid = routeid;
         this.routeid2 = UUID.randomUUID().toString();
     }
+*/
+
+    public FairOaiPmhRoute2(String oaiEndpoint, AbstractOpenAireProfile profile, AbstractOpenAireProfile fairProfile,
+                            ValidationJob validationJob, long maxNumberOfRecords, String routeid,
+                            ValidationJobRepository validationJobRepository,
+                            ValidationResultRepository validationResultRepository,
+                            SummaryValidationJobRepository summaryValidationJobRepository) {
+        this.oaiEndpoint = oaiEndpoint;
+        this.validationJob = validationJob;
+        this.profile = profile;
+        this.fairProfile = fairProfile;
+        this.maxNumberOfRecords = maxNumberOfRecords;
+        this.routeid = routeid;
+        this.routeid2 = UUID.randomUUID().toString();
+        this.validationJobRepository = validationJobRepository;
+        this.validationResultRepository = validationResultRepository;
+        this.summaryValidationJobRepository = summaryValidationJobRepository;
+    }
 
     @Override
     public void configure() throws Exception {
@@ -87,6 +114,10 @@ public class FairOaiPmhRoute2 extends RouteBuilder {
                 .handled(true)
                 .end();
 
+        onException(RejectedExecutionException.class)
+                .handled(true)
+                .log("RejectedExecutionException occurred. Suspending further processing.");
+
 
         from(oaiEndpoint)
                 .routeId("1")
@@ -121,8 +152,10 @@ public class FairOaiPmhRoute2 extends RouteBuilder {
                 .end()
                 .choice()
                     .when(header("MyHeader").isEqualTo("stop"))
-                        .process(new DataBaseProcessor())
-                        .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")//;.to("direct:saveToDatabase")
+                        .process(new SumUpNCleanProcessor(validationJobRepository, validationResultRepository, summaryValidationJobRepository,
+                        profile))
+                        /*.process(new DataBaseProcessor())
+                        .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")//;.to("direct:saveToDatabase")*/
                         .to("controlbus:route?routeId="+routeid+"&action=stop&async=true")
                     .endChoice()
                 .end();
@@ -130,7 +163,7 @@ public class FairOaiPmhRoute2 extends RouteBuilder {
         from("direct:fairProcessor")
                 .routeId(routeid2)
                 .split(xpath("//*[local-name()='record']"))
-                .process(new XmlProcessor(new FAIR_Data_GuidelinesProfile(), validationJob, maxNumberOfRecords))
+                .process(new XmlProcessor(fairProfile, validationJob, maxNumberOfRecords))
                 .choice()
                     .when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
                         .split(simple("${body[results]}"))
@@ -145,8 +178,10 @@ public class FairOaiPmhRoute2 extends RouteBuilder {
                 .end()
                 .choice()
                     .when(header("MyHeader").isEqualTo("stop"))
-                        .process(new DataBaseProcessor())
-                        .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")//;.to("direct:saveToDatabase")
+                        .process(new SumUpNCleanProcessor(validationJobRepository, validationResultRepository, summaryValidationJobRepository,
+                                fairProfile))
+                        /*.process(new DataBaseProcessor())
+                        .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")//;.to("direct:saveToDatabase")*/
                         .to("controlbus:route?routeId="+routeid2+"&action=stop&async=true")
                     .endChoice()
                 .end();

src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java

@@ -1,32 +1,21 @@
 package eu.dnetlib.validatorapi.routes;
 
+import eu.dnetlib.validatorapi.processors.SetsProcessor;
+import org.apache.camel.Exchange;
 import org.apache.camel.TypeConversionException;
 import org.apache.camel.builder.RouteBuilder;
 import org.apache.camel.impl.engine.DefaultShutdownStrategy;
+import org.apache.http.HttpStatus;
 import org.apache.http.client.ClientProtocolException;
 import org.springframework.stereotype.Component;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
 
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import java.io.StringReader;
-import java.io.StringWriter;
 import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.concurrent.TimeoutException;
 
 @Component
 public class OaiSetListRoute extends RouteBuilder {
 
+
     @Override
     public void configure() throws Exception {
 
@@ -56,88 +45,23 @@ public class OaiSetListRoute extends RouteBuilder {
                 .handled(true)
                 .end();
 
-        from("direct:getResponse")
-                .log("\n\n\n LIST ")
-                .end();
+        onException(TimeoutException.class)
+                .handled(true)
+                .process(exchange -> {
+                    exchange.getMessage().setHeader(Exchange.HTTP_RESPONSE_CODE, HttpStatus.SC_REQUEST_TIMEOUT);
+                    exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, "application/json");
+                    exchange.getIn().setBody("{\"error_code\":\"408\",\"message\":\"Unable to retrieve sets. Server response takes too long.\"}");
+                });
 
-        /*from("direct:getResponse")
+        from("direct:getResponse")
+                .routeId("myRoute")
                 .process(exchange -> {
                     String endpoint = exchange.getIn().getHeader("endpoint", String.class);
                     exchange.getIn().setHeader("dynamicEndpoint", endpoint);
-                })
-                .recipientList(header("dynamicEndpoint"))
-                .process(exchange -> {
-                    List<String> ListSetsXml = exchange.getIn().getBody(ArrayList.class);
-                    List<String> desiredJsonParts = new ArrayList<>();
-                    String xmlSets = "";
-
-                    // Extract the desired <set> elements from each XML string
-                    for (String xml : ListSetsXml) {
-                        List<String> extractedSets = extractSets(xml);
-                        for (String setXml : extractedSets)
-                            xmlSets += setXml;
-                    }
-
-                    JSONObject jsonObject = XML.toJSONObject(xmlSets);
-                    String jsonString = jsonObject.toString();
-                    exchange.getIn().setBody(jsonString);
-                })
-        .end();*/
-}
-
-         /*from("direct:processSets")
-                .process(exchange -> {
-                    // Access the individual response from the ArrayList
-                    ArrayList<?> responseList = exchange.getIn().getBody(ArrayList.class);
-                    InputStream responseStream = new ByteArrayInputStream(responseList.get(0).toString().getBytes());
-
-                    // Set the converted InputStream as the new body
-                    exchange.getIn().setBody(responseStream);
-                }).log("${body}")
-                .split(xpath("//*[local-name()='set']")).log("${body}")
-              //  .log("${body}")
-                .aggregate(constant(true), new ListSetAggregationStrategy())
-                .completionPredicate(exchangeProperty(Exchange.AGGREGATED_SIZE).isEqualTo(header("CamelSplitSize")))
-                .to("direct:processAggregatedSets")
-         .log("${body}");*/
-/*
-        from("direct:processAggregatedSets")
-                .process(exchange -> {
-                    String aggregatedXml = exchange.getIn().getBody(ArrayList.class) + "";
-                    JSONObject jsonObject = XML.toJSONObject(aggregatedXml);
-                    String jsonString = jsonObject.toString();
-                    exchange.getIn().setBody(jsonString);
-                }).to("seda:result");*/
-    //}
-
-    private List<String> extractSets(String xml) {
-        List<String> setXmls = new ArrayList<>();
-        try {
-            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-            DocumentBuilder builder = factory.newDocumentBuilder();
-            InputSource inputSource = new InputSource(new StringReader(xml));
-            Document document = builder.parse(inputSource);
-
-            NodeList setElements = document.getElementsByTagName("set");
-            for (int i = 0; i < setElements.getLength(); i++) {
-                Node setElement = setElements.item(i);
-                String setXml = nodeToString(setElement);
-                setXmls.add(setXml);
-            }
-        } catch (Exception e) {
-            // Handle any exceptions that occur during XML parsing or processing
-            log.error(e.getMessage());
-        }
-
-        return setXmls;
-    }
-
-    private String nodeToString(Node node) throws TransformerException {
-        StringWriter sw = new StringWriter();
-        TransformerFactory tf = TransformerFactory.newInstance();
-        Transformer transformer = tf.newTransformer();
-        transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
-        transformer.transform(new DOMSource(node), new StreamResult(sw));
-        return sw.toString();
+                }).recipientList(header("dynamicEndpoint"))
+                .parallelProcessing()
+                .timeout(20000)
+                .process(new SetsProcessor())
+                .end();
     }
 }

src/main/java/eu/dnetlib/validatorapi/routes/SimpleOaiPmhRoute.java

@@ -4,10 +4,13 @@ import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfi
 import eu.dnetlib.validatorapi.entities.ValidationIssue;
 import eu.dnetlib.validatorapi.entities.ValidationJob;
 import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
-import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
 import eu.dnetlib.validatorapi.processors.ErrorProcessor;
 import eu.dnetlib.validatorapi.processors.ExceptionProcessor;
+import eu.dnetlib.validatorapi.processors.SumUpNCleanProcessor;
 import eu.dnetlib.validatorapi.processors.XmlProcessor;
+import eu.dnetlib.validatorapi.repositories.SummaryValidationJobRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
+import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
 import org.apache.camel.TypeConversionException;
@@ -17,6 +20,7 @@ import org.apache.http.client.ClientProtocolException;
 
 import java.net.UnknownHostException;
 import java.util.Date;
+import java.util.concurrent.RejectedExecutionException;
 
 public class SimpleOaiPmhRoute extends RouteBuilder {
 
@@ -27,15 +31,24 @@ public class SimpleOaiPmhRoute extends RouteBuilder {
     private ValidationJob validationJob;
     String routeid;
 
+    private ValidationJobRepository validationJobRepository;
+    private ValidationResultRepository validationResultRepository;
+    private SummaryValidationJobRepository summaryValidationJobRepository;
 
 
     public SimpleOaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob,
-                             long maxNumberOfRecords, String routeid) {
+                             long maxNumberOfRecords, String routeid, ValidationJobRepository validationJobRepository,
+                             ValidationResultRepository validationResultRepository,
+                             SummaryValidationJobRepository summaryValidationJobRepository) {
         this.oaiEndpoint = oaiEndpoint;
         this.validationJob = validationJob;
         this.profile = profile;
         this.maxNumberOfRecords = maxNumberOfRecords;
         this.routeid = routeid;
+
+        this.validationResultRepository = validationResultRepository;
+        this.summaryValidationJobRepository = summaryValidationJobRepository;
+        this.validationJobRepository = validationJobRepository;
     }
 
     @Override
@@ -49,8 +62,7 @@ public class SimpleOaiPmhRoute extends RouteBuilder {
 
         onException(TypeConversionException.class)
                 .process(new ExceptionProcessor(validationJob))
-                .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true").log("\n\n\n\nHEREEEEEE")
-                .to("controlbus:route?routeId="+routeid+"&action=stop")
+                .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")
                 .maximumRedeliveries(0)
                 .handled(true)
                 .end();
@@ -58,7 +70,6 @@ public class SimpleOaiPmhRoute extends RouteBuilder {
         onException(ClientProtocolException.class)
                 .process(new ExceptionProcessor(validationJob))
                 .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")
-                .to("controlbus:route?routeId="+routeid+"&action=stop")
                 .maximumRedeliveries(0)
                 .handled(true)
                 .end();
@@ -66,7 +77,6 @@ public class SimpleOaiPmhRoute extends RouteBuilder {
         onException(UnknownHostException.class)
                 .process(new ExceptionProcessor(validationJob))
                 .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")
-                .to("controlbus:route?routeId="+routeid+"&action=stop")
                 .maximumRedeliveries(0)
                 .handled(true)
                 .end();
@@ -74,47 +84,48 @@ public class SimpleOaiPmhRoute extends RouteBuilder {
         onException(ClientProtocolException.class)
                 .process(new ExceptionProcessor(validationJob))
                 .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")
-                .to("controlbus:route?routeId="+routeid+"&action=stop")
                 .maximumRedeliveries(0)
                 .handled(true)
                 .end();
 
+        //TODO: Reconsider
+        onException(RejectedExecutionException.class)
+                .handled(true)  // Set 'handled' to true to prevent the exception from propagating
+                .log("RejectedExecutionException occurred. Suspending further processing.");
 
-/*
-        .process(new TypeConversionExceptionProcessor(validationJob)).log("\n\n\n\n HERE 2")
-                .process(new DataBaseProcessor())//.maximumRedeliveries(0)
-                .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true").log("\n\n\n\nHEREEEEEE")
-                .stop();
-*/
         from(oaiEndpoint)
                 .routeId(routeid)
                 .choice()
                     .when(xpath("//*[local-name()='record']"))
                         .split(xpath("//*[local-name()='record']"))
                         .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords))
-                        .choice()
+                            .choice()
                             .when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
-                            .split(simple("${body[results]}"))
-                            .to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true")
-                        .endChoice()
-                        .end()
-                        .choice()
+                              //  .log("SAVING RESULTS ${header.MyHeader}")
+                                .split(simple("${body[results]}"))
+                                .to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true")
+                                .endChoice()
+                            .end()
+                            .choice()
                             .when(simple("${body[issues]} && ${header.MyHeader} != 'stop'"))
-                            .split(simple("${body[issues]}"))
-                            .to("jpa:"+ ValidationIssue.class.getName()+ "?usePersist=true")
-                        .endChoice()
-                        .end()
-                        .choice()
-                            .when(header("MyHeader").isEqualTo("stop"))
-                            .process(new DataBaseProcessor())
-                            .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")//;.to("direct:saveToDatabase")
-                            .to("controlbus:route?routeId="+routeid+"&action=stop&async=true")
-                        .endChoice()
-                        .end()
-                .endChoice()
-                .otherwise()
-                    .process(new ErrorProcessor(validationJob))
-                    .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")
+                             //   .log("SAVING ISSUES ${header.MyHeader}")
+                                .split(simple("${body[issues]}"))
+                                .to("jpa:"+ ValidationIssue.class.getName()+ "?usePersist=true")
+                                .endChoice()
+                            .end()
+                            .choice()
+                                .when(simple("${header.MyHeader} == 'stop'"))
+                                .process(new SumUpNCleanProcessor(validationJobRepository, validationResultRepository, summaryValidationJobRepository,
+                                        profile))
+                                /*.process(new DataBaseProcessor())
+                                .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")//;.to("direct:saveToDatabase")*/
+                                .to("controlbus:route?routeId="+routeid+"&action=stop&async=true")
+                            .end()
+                    .endChoice()
+                    .otherwise()
+                        .process(new ErrorProcessor(validationJob))
+                        .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")
+                    .endChoice()
                 .end();
 
 /*