diff --git a/pom.xml b/pom.xml
index eb0e8f1..f175054 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,6 +99,15 @@
camel-http
${apache.camel.version}
+
+ org.apache.camel
+ camel-jpa
+ ${apache.camel.version}
+
+
+
+
+
diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
index d440224..ef88b80 100644
--- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
+++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
@@ -4,14 +4,12 @@ import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.openaire.*;
import eu.dnetlib.validatorapi.entities.RuleInfo;
-import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import eu.dnetlib.validatorapi.routes.OaiPmhRoute;
-import eu.dnetlib.validatorapi.routes.OaiSetListRoute;
import org.apache.camel.CamelContext;
import org.apache.camel.ProducerTemplate;
import org.apache.camel.builder.RouteBuilder;
@@ -124,7 +122,7 @@ public class ValidationController {
resultSum += engineResult.score();
validationResultRepository.save(validationRuleResult);
- saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult);
+ //saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult);
}
record++;
@@ -212,10 +210,15 @@ public class ValidationController {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
+
RouteBuilder oaiPmhRouteBuilder =
new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
- profile, validationJob, numberOfRecords, validationJobRepository,
- validationIssueRepository, validationResultRepository);
+ profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository);
+
+ /*
+ RouteBuilder oaiPmhRouteBuilder =
+ new OAI_PMH_RouteBuilder("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
+ profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository);*/
camelContext.addRoutes(oaiPmhRouteBuilder);
}
@@ -234,13 +237,9 @@ public class ValidationController {
public String getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
) throws Exception {
- RouteBuilder listSetsBuilder = new OaiSetListRoute();
ProducerTemplate producerTemplate = camelContext.createProducerTemplate();
String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);
-/*
- ConsumerTemplate consumerTemplate = camelContext.createConsumerTemplate();
- String result = consumerTemplate.receive("seda:result").getIn().getBody(String.class);
- return result;*/
+
return response;
/*
@@ -298,9 +297,9 @@ public class ValidationController {
return validationRuleResult;
}
- private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
+/* private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
for (String error:engineResult.errors()) {
- /*System.out.println("11111");*/
+ *//*System.out.println("11111");*//*
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
@@ -308,15 +307,15 @@ public class ValidationController {
validationIssue.issueType = "ERROR";
validationIssue.issueText = error;
validationIssueRepository.save(validationIssue);
-/*
+*//*
System.out.println(validationIssue);
-*/
+*//*
}
for (String warning: engineResult.warnings()){
-/*
+*//*
System.out.println("22222");
-*/
+*//*
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
@@ -324,13 +323,13 @@ public class ValidationController {
validationIssue.issueType = "WARNING";
validationIssue.issueText = warning;
validationIssueRepository.save(validationIssue);
-/*
+*//*
System.out.println(validationIssue);
-*/
+*//*
}
- }
+ }*/
private ValidationRuleResult constructValidationRuleResult(int validationJobId, String recordUrl, String ruleName,
AbstractOpenAireProfile profile, Guideline.Result engineResult) {
diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java
index c9fba9a..09f7344 100644
--- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java
+++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java
@@ -26,6 +26,16 @@ public class ValidationIssue implements Serializable {
@Column(name = "issue_type")
public String issueType;
+ public ValidationIssue(){};
+
+ public ValidationIssue(int validationJobId, String ruleName, String recordUrl, String issueText, String issueType) {
+ this.validationJobId = validationJobId;
+ this.ruleName = ruleName;
+ this.recordUrl = recordUrl;
+ this.issueText = issueText;
+ this.issueType = issueType;
+ }
+
public int getValidationJobId() {
return validationJobId;
}
diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java
index 0aca156..93de892 100644
--- a/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java
+++ b/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java
@@ -1,7 +1,6 @@
package eu.dnetlib.validatorapi.processors;
import eu.dnetlib.validatorapi.entities.ValidationJob;
-import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
@@ -10,13 +9,11 @@ import java.util.Date;
public class DataBaseProcessor implements Processor {
private final ValidationJob validationJob;
- private final ValidationJobRepository validationJobRepository;
private final ValidationResultRepository validationResultRepository;
- public DataBaseProcessor(ValidationJob validationJob, final ValidationJobRepository validationJobRepository,
+ public DataBaseProcessor(ValidationJob validationJob,
final ValidationResultRepository validationResultRepository){
this.validationJob = validationJob;
- this.validationJobRepository = validationJobRepository;
this.validationResultRepository = validationResultRepository;
}
@@ -26,7 +23,6 @@ public class DataBaseProcessor implements Processor {
validationJob.progress = "COMPLETED";
validationJob.endDate = new Date();
validationJob.status = validationResultRepository.getStatus(validationJob.id);
- System.out.println(validationJob.status);
- validationJobRepository.save(validationJob);
+ exchange.getIn().setBody(validationJob, ValidationJob.class);
}
}
diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java
index 05758ce..961e960 100644
--- a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java
+++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java
@@ -6,8 +6,6 @@ import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfi
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
-import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
-import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.w3c.dom.Document;
@@ -18,13 +16,15 @@ import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
public class XmlProcessor implements Processor {
private final AbstractOpenAireProfile profile;
private ValidationJob validationJob;
- private final ValidationIssueRepository validationIssueRepository;
- private final ValidationResultRepository validationResultRepository;
+ // private final ValidationIssueRepository validationIssueRepository;
private final long maxNumberOfRecords;
private int processedRecords;
// private int resultSum;
@@ -32,14 +32,10 @@ public class XmlProcessor implements Processor {
public XmlProcessor(final AbstractOpenAireProfile profile, ValidationJob validationJob,
- final ValidationIssueRepository validationIssueRepository,
- final ValidationResultRepository validationResultRepository,
final long maxNumberOfRecords){
super();
this.profile = profile;
this.validationJob = validationJob;
- this.validationIssueRepository = validationIssueRepository;
- this.validationResultRepository = validationResultRepository;
this.maxNumberOfRecords = maxNumberOfRecords;
}
@@ -51,6 +47,8 @@ public class XmlProcessor implements Processor {
final DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml)));
String recordUrl = extractRecordUrl(doc, "identifier");
+ List validationRuleResults = new ArrayList<>();
+ List validationIssues = new ArrayList<>();
if (profile != null) {
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
@@ -61,49 +59,45 @@ public class XmlProcessor implements Processor {
final Guideline.Result engineResult = (Guideline.Result) entry.getValue();
String ruleName = entry.getKey().toString();
validationRuleResult = new ValidationRuleResult();
- constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
+ constructValidationRuleResult(validationRuleResult, validationJob.id, profile.name(), recordUrl,
ruleName, profile, engineResult);
if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) {
validationRuleResult.hasErrors = true;
- saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR");
+ for (String error:engineResult.errors()) {
+ validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, error,"ERROR"));
+ }
}
if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) {
validationRuleResult.hasWarnings = true;
- saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING");
+ for (String warning:engineResult.warnings()) {
+ validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, warning, "WARNING"));
+ }
}
- validationResultRepository.save(validationRuleResult);
+ //validationResultRepository.save(validationRuleResult);
+ validationRuleResults.add(validationRuleResult);
scoreSum += engineResult.score();
-
}
+
}
processedRecords++;
validationJob.recordsTested = processedRecords;
validationJob.score = scoreSum/processedRecords;
- //System.out.println("\n\n\nvalidation job: " + validationJob.recordsTested);
- if (processedRecords > maxNumberOfRecords) {
+ Map resultsMap = new HashMap();
+ resultsMap.put("results", validationRuleResults);
+ resultsMap.put("issues", validationIssues);
+ exchange.getIn().setBody(resultsMap, Map.class);
+
+ if (processedRecords == maxNumberOfRecords) {
exchange.getIn().setHeader("MyHeader", "stop");
}
}
- private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName,
- Iterable issues, String issueType) {
- for (String issue:issues) {
- ValidationIssue validationIssue = new ValidationIssue();
- validationIssue.validationJobId = validationJobId;
- validationIssue.ruleName = ruleName;
- validationIssue.recordUrl = recordUrl;
- validationIssue.issueType = issueType;
- validationIssue.issueText = issue;
- validationIssueRepository.save(validationIssue);
- }
- }
-
//TODO consider throwing exception - UTIL class?
private String extractRecordUrl(final Document document, final String xmlField) {
NodeList identifierNodes = document.getElementsByTagName(xmlField);
@@ -115,10 +109,11 @@ public class XmlProcessor implements Processor {
}
- private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String recordUrl, String ruleName,
+ private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String guidelines, String recordUrl, String ruleName,
AbstractOpenAireProfile profile, Guideline.Result engineResult) {
validationRuleResult.validationJobId = validationJobId;
+ validationRuleResult.guidelines = guidelines;
validationRuleResult.ruleName = ruleName;
validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight();
validationRuleResult.recordUrl = recordUrl;
diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java
index 4713d31..7237eb1 100644
--- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java
+++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java
@@ -1,7 +1,9 @@
package eu.dnetlib.validatorapi.routes;
import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
+import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
+import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
import eu.dnetlib.validatorapi.processors.XmlProcessor;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
@@ -49,14 +51,24 @@ public class OaiPmhRoute extends RouteBuilder {
from(oaiEndpoint)
.routeId(date)
.split(xpath("//*[local-name()='record']"))
- .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository, maxNumberOfRecords))
- .choice().when(header("MyHeader").isEqualTo("stop"))
- .to("direct:saveToDatabase")
- .to("controlbus:route?routeId="+date+"&action=stop")
+ .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords))
+ .choice()
+ .when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
+ .split(simple("${body[results]}"))
+ .to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true")
+ .endChoice()
+ .when(simple("${body[issues]} && ${header.MyHeader} != 'stop'"))
+ .split(simple("${body[issues]}"))
+ .to("jpa:"+ ValidationIssue.class.getName()+ "?usePersist=true")
+ .endChoice()
+ .when(header("MyHeader").isEqualTo("stop"))
+ //.to("direct:saveToDatabase")
+ .to("controlbus:route?routeId="+date+"&action=stop")
.endChoice();
from("direct:saveToDatabase")
- .process(new DataBaseProcessor(validationJob, validationJobRepository, validationResultRepository));
+ .process(new DataBaseProcessor(validationJob, validationResultRepository))
+ .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true");
/*from("timer://myTimer?fixedRate=true&period=60000")
.routeId("OAIProcessingRoute")
diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java
index a0a1ef8..d30ab76 100644
--- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java
+++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java
@@ -25,35 +25,24 @@ import java.util.List;
@Component
public class OaiSetListRoute extends RouteBuilder {
- /*private final String oaiEndpoint;*/
-
- public String xml = new String();
-
-
- /*public OaiSetListRoute(final String oaiEndpoint) {
- this.oaiEndpoint = oaiEndpoint;
- }*/
-
-
@Override
public void configure() throws Exception {
from("direct:getResponse")
.process(exchange -> {
- // Get the endpoint URL from the request parameter
String endpoint = exchange.getIn().getHeader("endpoint", String.class);
- // Set the dynamic endpoint URL in the exchange header
exchange.getIn().setHeader("dynamicEndpoint", endpoint);
})
.recipientList(header("dynamicEndpoint"))
.process(exchange -> {
List ListSetsXml = exchange.getIn().getBody(ArrayList.class);
List desiredJsonParts = new ArrayList<>();
- String xmlSets ="";
+ String xmlSets = "";
+
// Extract the desired elements from each XML string
for (String xml : ListSetsXml) {
List extractedSets = extractSets(xml);
- for (String setXml:extractedSets)
+ for (String setXml : extractedSets)
xmlSets += setXml;
}
@@ -87,7 +76,7 @@ public class OaiSetListRoute extends RouteBuilder {
exchange.getIn().setBody(jsonString);
}).to("seda:result");*/
//}
-
+
private List extractSets(String xml) {
List setXmls = new ArrayList<>();
try {