MaDgIK
/
uoa-validator-api
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Steps to remove save methods from Processors. Added apache camel-jpa dependency

This commit is contained in:
Katerina 2023-07-14 14:21:21 +03:00
parent e1eb44ae73
commit ec549036d9
7 changed files with 84 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pom.xml
View File

@ -99,6 +99,15 @@
<artifactId>camel-http</artifactId>
<version>${apache.camel.version}</version>
</dependency>
<dependency>
<groupId>org.apache.camel</groupId>
<artifactId>camel-jpa</artifactId>
<version>${apache.camel.version}</version>
<!-- use the same version as your Camel core version -->
</dependency>
<!-- JSON -->
<dependency>

37
src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
View File

@ -4,14 +4,12 @@ import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.openaire.*;
import eu.dnetlib.validatorapi.entities.RuleInfo;
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import eu.dnetlib.validatorapi.routes.OaiPmhRoute;
import eu.dnetlib.validatorapi.routes.OaiSetListRoute;
import org.apache.camel.CamelContext;
import org.apache.camel.ProducerTemplate;
import org.apache.camel.builder.RouteBuilder;
@ -124,7 +122,7 @@ public class ValidationController {
resultSum += engineResult.score();
validationResultRepository.save(validationRuleResult);
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult);
//saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult);
}
record++;
@ -212,10 +210,15 @@ public class ValidationController {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
RouteBuilder oaiPmhRouteBuilder =
new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
profile, validationJob, numberOfRecords, validationJobRepository,
validationIssueRepository, validationResultRepository);
profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository);
/*
RouteBuilder oaiPmhRouteBuilder =
new OAI_PMH_RouteBuilder("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository);*/
camelContext.addRoutes(oaiPmhRouteBuilder);
}
@ -234,13 +237,9 @@ public class ValidationController {
public String getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
) throws Exception {
RouteBuilder listSetsBuilder = new OaiSetListRoute();
ProducerTemplate producerTemplate = camelContext.createProducerTemplate();
String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);
/*
ConsumerTemplate consumerTemplate = camelContext.createConsumerTemplate();
String result = consumerTemplate.receive("seda:result").getIn().getBody(String.class);
return result;*/
return response;
/*
@ -298,9 +297,9 @@ public class ValidationController {
return validationRuleResult;
}
private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
/* private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
for (String error:engineResult.errors()) {
/*System.out.println("11111");*/
*//*System.out.println("11111");*//*
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
@ -308,15 +307,15 @@ public class ValidationController {
validationIssue.issueType = "ERROR";
validationIssue.issueText = error;
validationIssueRepository.save(validationIssue);
/*
*//*
System.out.println(validationIssue);
*/
*//*
}
for (String warning: engineResult.warnings()){
/*
*//*
System.out.println("22222");
*/
*//*
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
@ -324,13 +323,13 @@ public class ValidationController {
validationIssue.issueType = "WARNING";
validationIssue.issueText = warning;
validationIssueRepository.save(validationIssue);
/*
*//*
System.out.println(validationIssue);
*/
*//*
}
}
}*/
private ValidationRuleResult constructValidationRuleResult(int validationJobId, String recordUrl, String ruleName,
AbstractOpenAireProfile profile, Guideline.Result engineResult) {

10
src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java
View File

@ -26,6 +26,16 @@ public class ValidationIssue implements Serializable {
@Column(name = "issue_type")
public String issueType;
public ValidationIssue(){};
public ValidationIssue(int validationJobId, String ruleName, String recordUrl, String issueText, String issueType) {
this.validationJobId = validationJobId;
this.ruleName = ruleName;
this.recordUrl = recordUrl;
this.issueText = issueText;
this.issueType = issueType;
}
public int getValidationJobId() {
return validationJobId;
}

8
src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.validatorapi.processors;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
@ -10,13 +9,11 @@ import java.util.Date;
public class DataBaseProcessor implements Processor {
private final ValidationJob validationJob;
private final ValidationJobRepository validationJobRepository;
private final ValidationResultRepository validationResultRepository;
public DataBaseProcessor(ValidationJob validationJob, final ValidationJobRepository validationJobRepository,
public DataBaseProcessor(ValidationJob validationJob,
final ValidationResultRepository validationResultRepository){
this.validationJob = validationJob;
this.validationJobRepository = validationJobRepository;
this.validationResultRepository = validationResultRepository;
}
@ -26,7 +23,6 @@ public class DataBaseProcessor implements Processor {
validationJob.progress = "COMPLETED";
validationJob.endDate = new Date();
validationJob.status = validationResultRepository.getStatus(validationJob.id);
System.out.println(validationJob.status);
validationJobRepository.save(validationJob);
exchange.getIn().setBody(validationJob, ValidationJob.class);
}
}

53
src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java
View File

@ -6,8 +6,6 @@ import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfi
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.w3c.dom.Document;
@ -18,13 +16,15 @@ import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class XmlProcessor implements Processor {
private final AbstractOpenAireProfile profile;
private ValidationJob validationJob;
private final ValidationIssueRepository validationIssueRepository;
private final ValidationResultRepository validationResultRepository;
// private final ValidationIssueRepository validationIssueRepository;
private final long maxNumberOfRecords;
private int processedRecords;
// private int resultSum;
@ -32,14 +32,10 @@ public class XmlProcessor implements Processor {
public XmlProcessor(final AbstractOpenAireProfile profile, ValidationJob validationJob,
final ValidationIssueRepository validationIssueRepository,
final ValidationResultRepository validationResultRepository,
final long maxNumberOfRecords){
super();
this.profile = profile;
this.validationJob = validationJob;
this.validationIssueRepository = validationIssueRepository;
this.validationResultRepository = validationResultRepository;
this.maxNumberOfRecords = maxNumberOfRecords;
}
@ -51,6 +47,8 @@ public class XmlProcessor implements Processor {
final DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml)));
String recordUrl = extractRecordUrl(doc, "identifier");
List<ValidationRuleResult> validationRuleResults = new ArrayList<>();
List<ValidationIssue> validationIssues = new ArrayList<>();
if (profile != null) {
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
@ -61,49 +59,45 @@ public class XmlProcessor implements Processor {
final Guideline.Result engineResult = (Guideline.Result) entry.getValue();
String ruleName = entry.getKey().toString();
validationRuleResult = new ValidationRuleResult();
constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
constructValidationRuleResult(validationRuleResult, validationJob.id, profile.name(), recordUrl,
ruleName, profile, engineResult);
if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) {
validationRuleResult.hasErrors = true;
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR");
for (String error:engineResult.errors()) {
validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, error,"ERROR"));
}
}
if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) {
validationRuleResult.hasWarnings = true;
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING");
for (String warning:engineResult.warnings()) {
validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, warning, "WARNING"));
}
}
validationResultRepository.save(validationRuleResult);
//validationResultRepository.save(validationRuleResult);
validationRuleResults.add(validationRuleResult);
scoreSum += engineResult.score();
}
}
processedRecords++;
validationJob.recordsTested = processedRecords;
validationJob.score = scoreSum/processedRecords;
//System.out.println("\n\n\nvalidation job: " + validationJob.recordsTested);
if (processedRecords > maxNumberOfRecords) {
Map<String, List> resultsMap = new HashMap();
resultsMap.put("results", validationRuleResults);
resultsMap.put("issues", validationIssues);
exchange.getIn().setBody(resultsMap, Map.class);
if (processedRecords == maxNumberOfRecords) {
exchange.getIn().setHeader("MyHeader", "stop");
}
}
private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName,
Iterable<String> issues, String issueType) {
for (String issue:issues) {
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
validationIssue.recordUrl = recordUrl;
validationIssue.issueType = issueType;
validationIssue.issueText = issue;
validationIssueRepository.save(validationIssue);
}
}
//TODO consider throwing exception - UTIL class?
private String extractRecordUrl(final Document document, final String xmlField) {
NodeList identifierNodes = document.getElementsByTagName(xmlField);
@ -115,10 +109,11 @@ public class XmlProcessor implements Processor {
}
private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String recordUrl, String ruleName,
private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String guidelines, String recordUrl, String ruleName,
AbstractOpenAireProfile profile, Guideline.Result engineResult) {
validationRuleResult.validationJobId = validationJobId;
validationRuleResult.guidelines = guidelines;
validationRuleResult.ruleName = ruleName;
validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight();
validationRuleResult.recordUrl = recordUrl;

22
src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java
View File

@ -1,7 +1,9 @@
package eu.dnetlib.validatorapi.routes;
import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
import eu.dnetlib.validatorapi.processors.XmlProcessor;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
@ -49,14 +51,24 @@ public class OaiPmhRoute extends RouteBuilder {
from(oaiEndpoint)
.routeId(date)
.split(xpath("//*[local-name()='record']"))
.process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository, maxNumberOfRecords))
.choice().when(header("MyHeader").isEqualTo("stop"))
.to("direct:saveToDatabase")
.to("controlbus:route?routeId="+date+"&action=stop")
.process(new XmlProcessor(profile, validationJob, maxNumberOfRecords))
.choice()
.when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
.split(simple("${body[results]}"))
.to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true")
.endChoice()
.when(simple("${body[issues]} && ${header.MyHeader} != 'stop'"))
.split(simple("${body[issues]}"))
.to("jpa:"+ ValidationIssue.class.getName()+ "?usePersist=true")
.endChoice()
.when(header("MyHeader").isEqualTo("stop"))
//.to("direct:saveToDatabase")
.to("controlbus:route?routeId="+date+"&action=stop")
.endChoice();
from("direct:saveToDatabase")
.process(new DataBaseProcessor(validationJob, validationJobRepository, validationResultRepository));
.process(new DataBaseProcessor(validationJob, validationResultRepository))
.to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true");
/*from("timer://myTimer?fixedRate=true&period=60000")
.routeId("OAIProcessingRoute")

19
src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java
View File

@ -25,35 +25,24 @@ import java.util.List;
@Component
public class OaiSetListRoute extends RouteBuilder {
/*private final String oaiEndpoint;*/
public String xml = new String();
/*public OaiSetListRoute(final String oaiEndpoint) {
this.oaiEndpoint = oaiEndpoint;
}*/
@Override
public void configure() throws Exception {
from("direct:getResponse")
.process(exchange -> {
// Get the endpoint URL from the request parameter
String endpoint = exchange.getIn().getHeader("endpoint", String.class);
// Set the dynamic endpoint URL in the exchange header
exchange.getIn().setHeader("dynamicEndpoint", endpoint);
})
.recipientList(header("dynamicEndpoint"))
.process(exchange -> {
List<String> ListSetsXml = exchange.getIn().getBody(ArrayList.class);
List<String> desiredJsonParts = new ArrayList<>();
String xmlSets ="";
String xmlSets = "";
// Extract the desired <set> elements from each XML string
for (String xml : ListSetsXml) {
List<String> extractedSets = extractSets(xml);
for (String setXml:extractedSets)
for (String setXml : extractedSets)
xmlSets += setXml;
}
@ -87,7 +76,7 @@ public class OaiSetListRoute extends RouteBuilder {
exchange.getIn().setBody(jsonString);
}).to("seda:result");*/
//}
private List<String> extractSets(String xml) {
List<String> setXmls = new ArrayList<>();
try {