MaDgIK
/
uoa-validator-api
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added separate processor to save in db + refined the APIs for the job and the error/warnings + added repository method for the status of a job

This commit is contained in:
Katerina 2023-06-13 15:57:39 +03:00
parent 8eae2632a1
commit 004b166b73
9 changed files with 115 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java
View File

@ -25,37 +25,37 @@ public class ReportController {
this.validationJobRepository = validationJobRepository;
this.validationResultRepository = validationResultRepository;
this.validationIssueRepository = validationIssueRepository;
}
}
@RequestMapping(value={"getJobResult"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public ValidationJob getJobResults(@RequestParam(name = "jobId") int jobId){
Optional<ValidationJob> validationJob = validationJobRepository.findById(jobId);
return (ValidationJob) validationJob.orElse(null);
}
@RequestMapping(value={"getJobResult"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public ValidationJob getJobResults(@RequestParam(name = "jobId") int jobId){
Optional<ValidationJob> validationJob = validationJobRepository.findById(jobId);
return (ValidationJob) validationJob.orElse(null);
}
@RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public List<SummaryResult> getSummaryJobResults(@RequestParam(name = "jobId") int jobId){
return validationResultRepository.getSummaryResult(jobId);
}
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrlsByRuleName(jobId, ruleName);
return extractRecordsGroupedByRule(resultList);
}
@RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrlsByRuleName(jobId, ruleName);
return extractRecordsGroupedByRule(resultList);
}
@RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public List<ValidationIssue> getRecordsByRule(@RequestParam(name = "jobId") int jobId,
@RequestParam(name = "ruleName") String ruleName){
return validationIssueRepository.getErrors(jobId, ruleName);
}
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId){
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId);
return extractRecordsGroupedByRule(resultList);
}
@RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId,
@RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId);
return extractRecordsGroupedByRule(resultList);
}
private Map<String, List<String>> extractRecordsGroupedByRule(List<Object[]> resultList) {
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();

27
src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
View File

@ -129,13 +129,13 @@ public class ValidationController {
System.out.println(record++);
}
}
validationJob.status = "COMPLETED";
validationJob.progress = "COMPLETED";
}
catch (Exception e) {
log.error("Validation job stopped unexpectedly." + e.getMessage());
System.out.println("ERROR " + e.getMessage());
validationJob.status = "STOPPED";
validationJob.progress = "STOPPED";
} finally {
validationJob.endDate = new Date();
@ -143,6 +143,7 @@ public class ValidationController {
validationJob.recordsTested = record;
validationJob.score = resultSum / record;
//TODO uncomment
log.info("Saving validation job " + validationJob.recordsTested);
validationJobRepository.save(validationJob);
}
@ -197,11 +198,9 @@ public class ValidationController {
ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords);
validationJob.guidelines = profile.name();
validationJob.status = "IN_PROGRESS";
validationJobRepository.save(validationJob);
log.info("Initial validation job id "+ validationJob.id);
int record = 0;
double resultSum = 0;
@ -211,24 +210,21 @@ public class ValidationController {
RouteBuilder oaiPmhRouteBuilder =
new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
profile, validationJob, numberOfRecords, validationIssueRepository, validationResultRepository);
profile, validationJob, numberOfRecords, validationJobRepository,
validationIssueRepository, validationResultRepository);
camelContext.addRoutes(oaiPmhRouteBuilder);
validationJob.status = "COMPLETED";
validationJob.progress = "COMPLETED";
}
catch (Exception e) {
log.error("Validation job stopped unexpectedly." + e.getMessage());
System.out.println("ERROR " + e.getMessage());
validationJob.status = "STOPPED";
} finally {
validationJob.progress = "STOPPED";
validationJob.endDate = new Date();
log.info("Final validation job "+ validationJob.id);
validationJob.recordsTested = record;
validationJob.score = resultSum / record;
validationJob.score = resultSum / validationJob.recordsTested;
validationJob.status = validationResultRepository.getStatus(validationJob.id);
validationJobRepository.save(validationJob);
}
}
@ -425,12 +421,13 @@ public class ValidationController {
}
record++;
}
validationJob.status = "COMPLETED";
validationJob.progress = "COMPLETED";
}
catch (Exception e) {
log.error("Validation job stopped unexpectedly." + e.getMessage());
validationJob.status = "STOPPED";
validationJob.progress = "STOPPED";
} finally {
validationJob.endDate = new Date();

14
src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java
View File

@ -8,7 +8,7 @@ import java.util.Date;
public class ValidationJob {
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Id
public int id; //not in use for now
public int id;
@Column(name = "base_url")
public String baseUrl;
@ -24,8 +24,10 @@ public class ValidationJob {
@Column(name = "records_tested")
public int recordsTested;
@Column(name="duration")
public String status; //stopped, completed, in progress
@Column(name="progress")
public String progress; //stopped, completed, in progress
@Column(name="status")
public String status; //success, failure
@Column(name="score")
public double score;
@ -42,7 +44,7 @@ public class ValidationJob {
@Override
public String toString() {
return "ValidationJob {" +
return "ValidationJob{" +
"id=" + id +
", baseUrl='" + baseUrl + '\'' +
", numberOfRecords=" + numberOfRecords +
@ -50,8 +52,12 @@ public class ValidationJob {
", startDate=" + startDate +
", endDate=" + endDate +
", recordsTested=" + recordsTested +
", progress='" + progress + '\'' +
", status='" + status + '\'' +
", score=" + score +
'}';
}
}

32
src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java Normal file
View File

@ -0,0 +1,32 @@
package eu.dnetlib.validatorapi.processors;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import java.util.Date;
public class DataBaseProcessor implements Processor {
private final ValidationJob validationJob;
private final ValidationJobRepository validationJobRepository;
private final ValidationResultRepository validationResultRepository;
public DataBaseProcessor(ValidationJob validationJob, final ValidationJobRepository validationJobRepository,
final ValidationResultRepository validationResultRepository){
this.validationJob = validationJob;
this.validationJobRepository = validationJobRepository;
this.validationResultRepository = validationResultRepository;
}
@Override
public void process(Exchange exchange) throws Exception {
validationJob.progress = "COMPLETED";
validationJob.endDate = new Date();
validationJob.status = validationResultRepository.getStatus(validationJob.id);
System.out.println(validationJob.status);
validationJobRepository.save(validationJob);
}
}

4
src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java
View File

@ -1,4 +0,0 @@
package eu.dnetlib.validatorapi.processors;
public class StopRouteProcessor {
}

12
src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java
View File

@ -22,14 +22,15 @@ import java.util.Map;
public class XmlProcessor implements Processor {
private final AbstractOpenAireProfile profile;
private final ValidationJob validationJob;
private ValidationJob validationJob;
private final ValidationIssueRepository validationIssueRepository;
private final ValidationResultRepository validationResultRepository;
private final long maxNumberOfRecords;
private long processedRecords;
private int processedRecords;
private int resultSum;
public XmlProcessor(final AbstractOpenAireProfile profile, final ValidationJob validationJob,
public XmlProcessor(final AbstractOpenAireProfile profile, ValidationJob validationJob,
final ValidationIssueRepository validationIssueRepository,
final ValidationResultRepository validationResultRepository,
final long maxNumberOfRecords){
@ -64,13 +65,16 @@ public class XmlProcessor implements Processor {
constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
ruleName, profile, engineResult);
validationResultRepository.save(validationRuleResult);
resultSum += engineResult.score();
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR");
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING");
}
}
processedRecords++;
exchange.setProperty("totalRecords", processedRecords);
validationJob.recordsTested = processedRecords;
validationJob.score = resultSum;
System.out.println("\n\n\nvalidation job: " + processedRecords);
if (processedRecords > maxNumberOfRecords) {
exchange.getIn().setHeader("MyHeader", "stop");

20
src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java
View File

@ -26,14 +26,26 @@ public interface ValidationIssueRepository extends JpaRepository<ValidationIssue
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
"FROM ValidationIssue vi " +
"WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" +
"WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id AND vi.ruleName=:ruleName)" +
"GROUP BY vi.issueText, vi.recordUrl")
List<Object[]> getAllErrorsRecordUrls(@Param("id") int validationJobId);
List<Object[]> getAllErrorsRecordUrlsByRuleName(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
"FROM ValidationIssue vi " +
"WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id)" +
"WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id AND vi.ruleName=:ruleName)" +
"GROUP BY vi.issueText, vi.recordUrl")
List<Object[]> getAllWarningsRecordUrls(@Param("id") int validationJobId);
List<Object[]> getAllWarningsRecordUrlsByRuleName(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
@Query(value = "SELECT vi.ruleName, vi.issueText, vi.recordUrl " +
"FROM ValidationIssue vi " +
"WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" +
"GROUP BY vi.ruleName, vi.issueText, vi.recordUrl")
List<Object[]> getAllErrors(@Param("id") int validationJobId);
@Query(value = "SELECT vi.ruleName, vi.issueText, vi.recordUrl " +
"FROM ValidationIssue vi " +
"WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id)" +
"GROUP BY vi.ruleName, vi.issueText, vi.recordUrl")
List<Object[]> getAllWarnings(@Param("id") int validationJobId);
}

6
src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java
View File

@ -19,4 +19,10 @@ public interface ValidationResultRepository extends JpaRepository<ValidationRule
"FROM ValidationRuleResult sr " + "WHERE sr.validationJobId = :id " +
"GROUP BY sr.ruleName, sr.ruleWeight")
List<SummaryResult> getSummaryResult(@Param("id") int validationJobId);
@Query("SELECT CASE WHEN COUNT(vr) > 0 THEN 'FAILURE' ELSE 'SUCCESS' END " +
"FROM ValidationRuleResult vr " +
"WHERE vr.validationJobId = :id AND vr.status = 'FAILURE'")
public String getStatus(@Param("id") int validationJobId);
}

14
src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java
View File

@ -2,8 +2,10 @@ package eu.dnetlib.validatorapi.routes;
import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
import eu.dnetlib.validatorapi.processors.XmlProcessor;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
@ -19,18 +21,21 @@ public class OaiPmhRoute extends RouteBuilder {
private AbstractOpenAireProfile profile;
private long maxNumberOfRecords = 0;
private ValidationJob validationJob;
private final ValidationJobRepository validationJobRepository;
private final ValidationIssueRepository validationIssueRepository;
private final ValidationResultRepository validationResultRepository;
CountDownLatch latch;
public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob,
long maxNumberOfRecords, final ValidationIssueRepository validationIssueRepository,
long maxNumberOfRecords, final ValidationJobRepository validationJobRepository,
final ValidationIssueRepository validationIssueRepository,
final ValidationResultRepository validationResultRepository) {
this.oaiEndpoint = oaiEndpoint;
this.validationJob = validationJob;
this.profile = profile;
this.maxNumberOfRecords = maxNumberOfRecords;
this.validationJobRepository = validationJobRepository;
this.validationIssueRepository = validationIssueRepository;
this.validationResultRepository = validationResultRepository;
this.latch = latch;
@ -41,13 +46,18 @@ public class OaiPmhRoute extends RouteBuilder {
String date = new Date().toString();
from(oaiEndpoint).routeId(date)
from(oaiEndpoint)
.routeId(date)
.split(xpath("//*[local-name()='record']"))
.process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository, maxNumberOfRecords))
.choice().when(header("MyHeader").isEqualTo("stop"))
.to("direct:saveToDatabase")
.to("controlbus:route?routeId="+date+"&action=stop")
.endChoice();
from("direct:saveToDatabase")
.process(new DataBaseProcessor(validationJob, validationJobRepository, validationResultRepository));
/*from("timer://myTimer?fixedRate=true&period=60000")
.routeId("OAIProcessingRoute")
.setHeader("MyHeader", constant("stop"))