MaDgIK
/
uoa-validator-api
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enhancing all APIs to get all the information needed for Job result and Rule warnings and errors + record urls

This commit is contained in:
Katerina 2023-06-20 15:42:16 +03:00
parent 004b166b73
commit ce4d729185
9 changed files with 215 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java
View File

@ -1,7 +1,7 @@
package eu.dnetlib.validatorapi.controllers;
import eu.dnetlib.validatorapi.entities.IssueDescriptor;
import eu.dnetlib.validatorapi.entities.SummaryResult;
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
@ -9,7 +9,9 @@ import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@RestController
@CrossOrigin(origins = "*")
@ -35,9 +37,10 @@ public class ReportController {
@RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public List<SummaryResult> getSummaryJobResults(@RequestParam(name = "jobId") int jobId){
return validationResultRepository.getSummaryResult(jobId);
return validationResultRepository.getFullSummaryResult(jobId);
}
/*
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrlsByRuleName(jobId, ruleName);
@ -50,12 +53,6 @@ public class ReportController {
return extractRecordsGroupedByRule(resultList);
}
@RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public List<ValidationIssue> getRecordsByRule(@RequestParam(name = "jobId") int jobId,
@RequestParam(name = "ruleName") String ruleName){
return validationIssueRepository.getErrors(jobId, ruleName);
}
private Map<String, List<String>> extractRecordsGroupedByRule(List<Object[]> resultList) {
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
@ -73,4 +70,47 @@ public class ReportController {
return recordUrlsByIssueText;
}
}
*/
@RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
private List<IssueDescriptor> getWarningDescriptors(@RequestParam(name = "jobId") int jobId,
@RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrlsByRuleName(jobId, ruleName);
return extractIssueDescriptor(resultList);
}
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
private List<IssueDescriptor> getErrorDescriptors(@RequestParam(name = "jobId") int jobId,
@RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrlsByRuleName(jobId, ruleName);
return extractIssueDescriptor(resultList);
}
private List<IssueDescriptor> extractIssueDescriptor(List<Object[]> resultList) {
List<IssueDescriptor> issueDescriptors = new ArrayList<>();
for (Object[] result : resultList) {
String issueText = (String) result[0];
String recordUrl = (String) result[1];
IssueDescriptor issueDescriptor = null;
for (IssueDescriptor descriptor : issueDescriptors) {
if (descriptor.getDescription().equals(issueText)) {
issueDescriptor = descriptor;
break;
}
}
if (issueDescriptor == null) {
issueDescriptor = new IssueDescriptor();
issueDescriptor.setDescription(issueText);
issueDescriptors.add(issueDescriptor);
}
issueDescriptor.getRecords().add(recordUrl);
}
return issueDescriptors;
}
}

29
src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
View File

@ -11,11 +11,13 @@ import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import eu.dnetlib.validatorapi.routes.OaiPmhRoute;
import eu.dnetlib.validatorapi.routes.OaiSetListRoute;
import org.apache.camel.CamelContext;
import org.apache.camel.builder.RouteBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@ -208,12 +210,16 @@ public class ValidationController {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
RouteBuilder oaiPmhRouteBuilder =
RouteBuilder oaiPmhRouteBuilder =
new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
profile, validationJob, numberOfRecords, validationJobRepository,
validationIssueRepository, validationResultRepository);
camelContext.addRoutes(oaiPmhRouteBuilder);
validationJob.progress = "COMPLETED";
/* RouteBuilder listSetsBuilder = new OaiSetListRoute("oaipmh://"+ baseURL + "?verb=ListSets");
camelContext.addRoutes(listSetsBuilder);
validationJob.progress = "COMPLETED";*/
}
catch (Exception e) {
@ -227,6 +233,25 @@ public class ValidationController {
}
}
@RequestMapping(value = {"/getSets"}, method = RequestMethod.GET)
public void getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
) throws Exception {
RouteBuilder listSetsBuilder = new OaiSetListRoute("oaipmh://"+ baseURL + "?verb=ListSets");
camelContext.addRoutes(listSetsBuilder);
camelContext.getRouteController().startRoute("oaiSetListRoute");
camelContext.getRouteController().stopRoute("oaiSetListRoute");
System.out.println(((OaiSetListRoute) listSetsBuilder).xml);
}
@RequestMapping(value = {"/demo"}, produces = MediaType.APPLICATION_JSON_VALUE, method = RequestMethod.POST)
public String demo() {
// Logic to retrieve sets and convert to JSON
// Replace with your actual logic
// Example response
String jsonResponse = "{ \"sets\": [\"Set1\", \"Set2\", \"Set3\"] }";
return jsonResponse;
}
private void constructValidationJobResult(ValidationJob validationJob, ValidationRuleResult validationRuleResult,
Map.Entry entry, AbstractOpenAireProfile profile, Document document) {

25
src/main/java/eu/dnetlib/validatorapi/entities/IssueDescriptor.java Normal file
View File

@ -0,0 +1,25 @@
package eu.dnetlib.validatorapi.entities;
import java.util.ArrayList;
import java.util.List;
public class IssueDescriptor {
private String description;
private List<String> records = new ArrayList<>();
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public List<String> getRecords() {
return records;
}
public void setRecords(List<String> records) {
this.records = records;
}
}

37
src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java
View File

@ -9,8 +9,11 @@ public class SummaryResult {
@Id
String rule_name;
int rule_weight;
String rule_status;
long passed_records;
long failed_records;
boolean has_errors;
boolean has_warnings;
public SummaryResult(String rule_name, long passed_records, long failed_records) {
this.rule_name = rule_name;
@ -25,6 +28,16 @@ public class SummaryResult {
this.failed_records = failed_records;
}
public SummaryResult(String rule_name, int rule_weight, long passed_records, long failed_records, String rule_status, boolean has_errors, boolean has_warnings) {
this.rule_name = rule_name;
this.rule_weight = rule_weight;
this.rule_status = rule_status;
this.passed_records = passed_records;
this.failed_records = failed_records;
this.has_errors = has_errors;
this.has_warnings = has_warnings;
}
public String getRule_name() {
return rule_name;
}
@ -57,6 +70,30 @@ public class SummaryResult {
this.failed_records = failed_records;
}
public String getRule_status() {
return rule_status;
}
public void setRule_status(String rule_status) {
this.rule_status = rule_status;
}
public boolean isHas_errors() {
return has_errors;
}
public void setHas_errors(boolean has_errors) {
this.has_errors = has_errors;
}
public boolean isHas_warnings() {
return has_warnings;
}
public void setHas_warnings(boolean has_warnings) {
this.has_warnings = has_warnings;
}
@Override
public String toString() {
return "SummaryResult{" +

5
src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java
View File

@ -24,10 +24,13 @@ public class ValidationJob {
@Column(name = "records_tested")
public int recordsTested;
@Column(name="progress")
public String progress; //stopped, completed, in progress
@Column(name="status")
public String status; //success, failure
@Column(name="score")
public double score;
@ -39,7 +42,7 @@ public class ValidationJob {
this.startDate = new Date();
this.baseUrl = baseUrl;
this.numberOfRecords = numberOfRecords;
this.status = "IN_PROGRESS";
this.progress = "IN_PROGRESS";
}
@Override

12
src/main/java/eu/dnetlib/validatorapi/entities/ValidationRuleResult.java
View File

@ -33,19 +33,27 @@ public class ValidationRuleResult implements Serializable {
@Column(name = "score")
public double score;
@Column(name = "has_errors")
public boolean hasErrors;
@Column(name = "hasWarnings")
public boolean hasWarnings;
public ValidationRuleResult() {}
@Override
public String toString() {
return "ValidationJobResult{" +
return "ValidationRuleResult{" +
"validationJobId=" + validationJobId +
", ruleName='" + ruleName + '\'' +
", ruleWeight='" + ruleWeight + '\'' +
", ruleWeight=" + ruleWeight +
", recordUrl='" + recordUrl + '\'' +
", internalError='" + internalError + '\'' +
", status='" + status + '\'' +
", score=" + score +
", hasErrors=" + hasErrors +
", hasWarnings=" + hasWarnings +
'}';
}
}

17
src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java
View File

@ -53,8 +53,6 @@ public class XmlProcessor implements Processor {
if (profile != null) {
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
System.out.println("DC IDENTIFIER: " + recordUrl);
Map<String, Guideline.Result> results = validationResult.results();
ValidationRuleResult validationRuleResult;
@ -64,17 +62,26 @@ public class XmlProcessor implements Processor {
validationRuleResult = new ValidationRuleResult();
constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
ruleName, profile, engineResult);
if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) {
validationRuleResult.hasErrors = true;
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR");
}
if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) {
validationRuleResult.hasWarnings = true;
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING");
}
validationResultRepository.save(validationRuleResult);
resultSum += engineResult.score();
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR");
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING");
}
}
processedRecords++;
validationJob.recordsTested = processedRecords;
validationJob.score = resultSum;
System.out.println("\n\n\nvalidation job: " + processedRecords);
//System.out.println("\n\n\nvalidation job: " + validationJob.recordsTested);
if (processedRecords > maxNumberOfRecords) {
exchange.getIn().setHeader("MyHeader", "stop");

16
src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java
View File

@ -22,7 +22,21 @@ public interface ValidationResultRepository extends JpaRepository<ValidationRule
@Query("SELECT CASE WHEN COUNT(vr) > 0 THEN 'FAILURE' ELSE 'SUCCESS' END " +
"FROM ValidationRuleResult vr " +
"WHERE vr.validationJobId = :id AND vr.status = 'FAILURE'")
"WHERE vr.validationJobId = :id AND vr.status = 'FAILURE' OR vr.internalError != null")
public String getStatus(@Param("id") int validationJobId);
@Query(value = "SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," +
"COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passedRecords," +
"COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failedRecords," +
"(SELECT CASE WHEN COUNT(DISTINCT CASE WHEN vr.status = 'FAILURE' THEN vr.ruleName END) = COUNT(DISTINCT vr.ruleName) THEN 'FAILURE' ELSE 'SUCCESS' END " +
"FROM eu.dnetlib.validatorapi.entities.ValidationRuleResult vr WHERE vr.ruleName = sr.ruleName)," +
"CASE WHEN COUNT(CASE WHEN has_errors IS TRUE THEN 1 END) > 0 THEN TRUE ELSE FALSE END AS hasErrors," +
"CASE WHEN COUNT(CASE WHEN has_warnings IS TRUE THEN 1 END) > 0 THEN TRUE ELSE FALSE END AS hasWarnings)" +
"FROM eu.dnetlib.validatorapi.entities.ValidationRuleResult sr " +
"WHERE sr.validationJobId = :id " +
"GROUP BY sr.ruleName, sr.ruleWeight")
List<SummaryResult> getFullSummaryResult(@Param("id") int validationJobId);
}

35
src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java Normal file
View File

@ -0,0 +1,35 @@
package eu.dnetlib.validatorapi.routes;
import org.apache.camel.builder.RouteBuilder;
public class OaiSetListRoute extends RouteBuilder {
private final String oaiEndpoint;
public String xml = new String();
public OaiSetListRoute(final String oaiEndpoint) {
this.oaiEndpoint = oaiEndpoint;
}
@Override
public void configure() throws Exception {
/* from(oaiEndpoint).process(exchange -> {
// System.out.println(exchange.getIn().getBody(String.class));
xml+=exchange.getIn().getBody(String.class)+"\n\n Katerina \n\n";
});*/
from(oaiEndpoint)
.convertBodyTo(String.class) // Convert response to String
.marshal().json() // Convert String to JSON
.to("direct:serveResponse")
.log("JSON Response: ${body}");
// Serve the JSON response
from("direct:serveResponse")
.setHeader("Content-Type", constant("application/json")) // Set the response content type
.setHeader("CamelHttpResponseCode", constant(200)) // Set the HTTP response code
.setBody(simple("${body}")) // Set the response body
.log("${body}")
.to("mock:result"); // Replace 'mock:result' with the appropriate endpoint to serve the response
}
}