From ce4d729185b99f9bb13fe956d103eb1ad1bf9f42 Mon Sep 17 00:00:00 2001 From: Katerina Date: Tue, 20 Jun 2023 15:42:16 +0300 Subject: [PATCH] Enhancing all APIs to get all the information needed for Job result and Rule warnings and errors + record urls --- .../controllers/ReportController.java | 60 +++++++++++++++---- .../controllers/ValidationController.java | 29 ++++++++- .../entities/IssueDescriptor.java | 25 ++++++++ .../validatorapi/entities/SummaryResult.java | 37 ++++++++++++ .../validatorapi/entities/ValidationJob.java | 5 +- .../entities/ValidationRuleResult.java | 12 +++- .../validatorapi/processors/XmlProcessor.java | 17 ++++-- .../ValidationResultRepository.java | 16 ++++- .../validatorapi/routes/OaiSetListRoute.java | 35 +++++++++++ 9 files changed, 215 insertions(+), 21 deletions(-) create mode 100644 src/main/java/eu/dnetlib/validatorapi/entities/IssueDescriptor.java create mode 100644 src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java index bac6066..1a953d9 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java @@ -1,7 +1,7 @@ package eu.dnetlib.validatorapi.controllers; +import eu.dnetlib.validatorapi.entities.IssueDescriptor; import eu.dnetlib.validatorapi.entities.SummaryResult; -import eu.dnetlib.validatorapi.entities.ValidationIssue; import eu.dnetlib.validatorapi.entities.ValidationJob; import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; @@ -9,7 +9,9 @@ import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; @RestController @CrossOrigin(origins = "*") @@ -35,9 +37,10 @@ public class ReportController { @RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public List getSummaryJobResults(@RequestParam(name = "jobId") int jobId){ - return validationResultRepository.getSummaryResult(jobId); + return validationResultRepository.getFullSummaryResult(jobId); } + /* @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public Map> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ List resultList = validationIssueRepository.getAllErrorsRecordUrlsByRuleName(jobId, ruleName); @@ -50,12 +53,6 @@ public class ReportController { return extractRecordsGroupedByRule(resultList); } - @RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public List getRecordsByRule(@RequestParam(name = "jobId") int jobId, - @RequestParam(name = "ruleName") String ruleName){ - return validationIssueRepository.getErrors(jobId, ruleName); - } - private Map> extractRecordsGroupedByRule(List resultList) { Map> recordUrlsByIssueText = new HashMap<>(); @@ -73,4 +70,47 @@ public class ReportController { return recordUrlsByIssueText; } -} + */ + + @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + private List getWarningDescriptors(@RequestParam(name = "jobId") int jobId, + @RequestParam(name = "ruleName") String ruleName){ + List resultList = validationIssueRepository.getAllWarningsRecordUrlsByRuleName(jobId, ruleName); + return extractIssueDescriptor(resultList); + } + + + @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + private List getErrorDescriptors(@RequestParam(name = "jobId") int jobId, + @RequestParam(name = "ruleName") String ruleName){ + List resultList = validationIssueRepository.getAllErrorsRecordUrlsByRuleName(jobId, ruleName); + return extractIssueDescriptor(resultList); + } + + private List extractIssueDescriptor(List resultList) { + List issueDescriptors = new ArrayList<>(); + + for (Object[] result : resultList) { + String issueText = (String) result[0]; + String recordUrl = (String) result[1]; + + IssueDescriptor issueDescriptor = null; + for (IssueDescriptor descriptor : issueDescriptors) { + if (descriptor.getDescription().equals(issueText)) { + issueDescriptor = descriptor; + break; + } + } + + if (issueDescriptor == null) { + issueDescriptor = new IssueDescriptor(); + issueDescriptor.setDescription(issueText); + issueDescriptors.add(issueDescriptor); + } + + issueDescriptor.getRecords().add(recordUrl); + } + + return issueDescriptors; + } +} \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index 44e09b1..75b44df 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -11,11 +11,13 @@ import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import eu.dnetlib.validatorapi.routes.OaiPmhRoute; +import eu.dnetlib.validatorapi.routes.OaiSetListRoute; import org.apache.camel.CamelContext; import org.apache.camel.builder.RouteBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -208,12 +210,16 @@ public class ValidationController { DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); - RouteBuilder oaiPmhRouteBuilder = + RouteBuilder oaiPmhRouteBuilder = new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix , profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository); camelContext.addRoutes(oaiPmhRouteBuilder); - validationJob.progress = "COMPLETED"; + + + /* RouteBuilder listSetsBuilder = new OaiSetListRoute("oaipmh://"+ baseURL + "?verb=ListSets"); + camelContext.addRoutes(listSetsBuilder); + validationJob.progress = "COMPLETED";*/ } catch (Exception e) { @@ -227,6 +233,25 @@ public class ValidationController { } } + @RequestMapping(value = {"/getSets"}, method = RequestMethod.GET) + public void getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now + ) throws Exception { + RouteBuilder listSetsBuilder = new OaiSetListRoute("oaipmh://"+ baseURL + "?verb=ListSets"); + camelContext.addRoutes(listSetsBuilder); + camelContext.getRouteController().startRoute("oaiSetListRoute"); + camelContext.getRouteController().stopRoute("oaiSetListRoute"); + System.out.println(((OaiSetListRoute) listSetsBuilder).xml); + } + @RequestMapping(value = {"/demo"}, produces = MediaType.APPLICATION_JSON_VALUE, method = RequestMethod.POST) + public String demo() { + // Logic to retrieve sets and convert to JSON + // Replace with your actual logic + + // Example response + String jsonResponse = "{ \"sets\": [\"Set1\", \"Set2\", \"Set3\"] }"; + + return jsonResponse; + } private void constructValidationJobResult(ValidationJob validationJob, ValidationRuleResult validationRuleResult, Map.Entry entry, AbstractOpenAireProfile profile, Document document) { diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/IssueDescriptor.java b/src/main/java/eu/dnetlib/validatorapi/entities/IssueDescriptor.java new file mode 100644 index 0000000..8716218 --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/entities/IssueDescriptor.java @@ -0,0 +1,25 @@ +package eu.dnetlib.validatorapi.entities; + +import java.util.ArrayList; +import java.util.List; + +public class IssueDescriptor { + private String description; + private List records = new ArrayList<>(); + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } +} diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java b/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java index 7f75821..3f936c1 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java @@ -9,8 +9,11 @@ public class SummaryResult { @Id String rule_name; int rule_weight; + String rule_status; long passed_records; long failed_records; + boolean has_errors; + boolean has_warnings; public SummaryResult(String rule_name, long passed_records, long failed_records) { this.rule_name = rule_name; @@ -25,6 +28,16 @@ public class SummaryResult { this.failed_records = failed_records; } + public SummaryResult(String rule_name, int rule_weight, long passed_records, long failed_records, String rule_status, boolean has_errors, boolean has_warnings) { + this.rule_name = rule_name; + this.rule_weight = rule_weight; + this.rule_status = rule_status; + this.passed_records = passed_records; + this.failed_records = failed_records; + this.has_errors = has_errors; + this.has_warnings = has_warnings; + } + public String getRule_name() { return rule_name; } @@ -57,6 +70,30 @@ public class SummaryResult { this.failed_records = failed_records; } + public String getRule_status() { + return rule_status; + } + + public void setRule_status(String rule_status) { + this.rule_status = rule_status; + } + + public boolean isHas_errors() { + return has_errors; + } + + public void setHas_errors(boolean has_errors) { + this.has_errors = has_errors; + } + + public boolean isHas_warnings() { + return has_warnings; + } + + public void setHas_warnings(boolean has_warnings) { + this.has_warnings = has_warnings; + } + @Override public String toString() { return "SummaryResult{" + diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java index e12be6b..ede8171 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java @@ -24,10 +24,13 @@ public class ValidationJob { @Column(name = "records_tested") public int recordsTested; + @Column(name="progress") public String progress; //stopped, completed, in progress + @Column(name="status") public String status; //success, failure + @Column(name="score") public double score; @@ -39,7 +42,7 @@ public class ValidationJob { this.startDate = new Date(); this.baseUrl = baseUrl; this.numberOfRecords = numberOfRecords; - this.status = "IN_PROGRESS"; + this.progress = "IN_PROGRESS"; } @Override diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationRuleResult.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationRuleResult.java index 186da1b..94562aa 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationRuleResult.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationRuleResult.java @@ -33,19 +33,27 @@ public class ValidationRuleResult implements Serializable { @Column(name = "score") public double score; + @Column(name = "has_errors") + public boolean hasErrors; + + @Column(name = "hasWarnings") + public boolean hasWarnings; + public ValidationRuleResult() {} @Override public String toString() { - return "ValidationJobResult{" + + return "ValidationRuleResult{" + "validationJobId=" + validationJobId + ", ruleName='" + ruleName + '\'' + - ", ruleWeight='" + ruleWeight + '\'' + + ", ruleWeight=" + ruleWeight + ", recordUrl='" + recordUrl + '\'' + ", internalError='" + internalError + '\'' + ", status='" + status + '\'' + ", score=" + score + + ", hasErrors=" + hasErrors + + ", hasWarnings=" + hasWarnings + '}'; } } diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java index a901394..0c3b154 100644 --- a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java +++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java @@ -53,8 +53,6 @@ public class XmlProcessor implements Processor { if (profile != null) { XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that? - System.out.println("DC IDENTIFIER: " + recordUrl); - Map results = validationResult.results(); ValidationRuleResult validationRuleResult; @@ -64,17 +62,26 @@ public class XmlProcessor implements Processor { validationRuleResult = new ValidationRuleResult(); constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl, ruleName, profile, engineResult); + + if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) { + validationRuleResult.hasErrors = true; + saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR"); + } + + if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) { + validationRuleResult.hasWarnings = true; + saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING"); + } + validationResultRepository.save(validationRuleResult); resultSum += engineResult.score(); - saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR"); - saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING"); } } processedRecords++; validationJob.recordsTested = processedRecords; validationJob.score = resultSum; - System.out.println("\n\n\nvalidation job: " + processedRecords); + //System.out.println("\n\n\nvalidation job: " + validationJob.recordsTested); if (processedRecords > maxNumberOfRecords) { exchange.getIn().setHeader("MyHeader", "stop"); diff --git a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java index 46fe887..f8864c4 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java @@ -22,7 +22,21 @@ public interface ValidationResultRepository extends JpaRepository 0 THEN 'FAILURE' ELSE 'SUCCESS' END " + "FROM ValidationRuleResult vr " + - "WHERE vr.validationJobId = :id AND vr.status = 'FAILURE'") + "WHERE vr.validationJobId = :id AND vr.status = 'FAILURE' OR vr.internalError != null") public String getStatus(@Param("id") int validationJobId); + @Query(value = "SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," + + "COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passedRecords," + + "COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failedRecords," + + "(SELECT CASE WHEN COUNT(DISTINCT CASE WHEN vr.status = 'FAILURE' THEN vr.ruleName END) = COUNT(DISTINCT vr.ruleName) THEN 'FAILURE' ELSE 'SUCCESS' END " + + "FROM eu.dnetlib.validatorapi.entities.ValidationRuleResult vr WHERE vr.ruleName = sr.ruleName)," + + "CASE WHEN COUNT(CASE WHEN has_errors IS TRUE THEN 1 END) > 0 THEN TRUE ELSE FALSE END AS hasErrors," + + "CASE WHEN COUNT(CASE WHEN has_warnings IS TRUE THEN 1 END) > 0 THEN TRUE ELSE FALSE END AS hasWarnings)" + + "FROM eu.dnetlib.validatorapi.entities.ValidationRuleResult sr " + + "WHERE sr.validationJobId = :id " + + "GROUP BY sr.ruleName, sr.ruleWeight") + List getFullSummaryResult(@Param("id") int validationJobId); + + + } diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java new file mode 100644 index 0000000..ade1f18 --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java @@ -0,0 +1,35 @@ +package eu.dnetlib.validatorapi.routes; + +import org.apache.camel.builder.RouteBuilder; + +public class OaiSetListRoute extends RouteBuilder { + + private final String oaiEndpoint; + + public String xml = new String(); + public OaiSetListRoute(final String oaiEndpoint) { + this.oaiEndpoint = oaiEndpoint; + } + + @Override + public void configure() throws Exception { + /* from(oaiEndpoint).process(exchange -> { + // System.out.println(exchange.getIn().getBody(String.class)); + xml+=exchange.getIn().getBody(String.class)+"\n\n Katerina \n\n"; + });*/ + + from(oaiEndpoint) + .convertBodyTo(String.class) // Convert response to String + .marshal().json() // Convert String to JSON + .to("direct:serveResponse") + .log("JSON Response: ${body}"); + + // Serve the JSON response + from("direct:serveResponse") + .setHeader("Content-Type", constant("application/json")) // Set the response content type + .setHeader("CamelHttpResponseCode", constant(200)) // Set the HTTP response code + .setBody(simple("${body}")) // Set the response body + .log("${body}") + .to("mock:result"); // Replace 'mock:result' with the appropriate endpoint to serve the response + } +}