progress with apache camel dynamic routes
This commit is contained in:
parent
a15670b741
commit
fe7a8b1631
14
pom.xml
14
pom.xml
|
@ -22,6 +22,7 @@
|
||||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||||
<timestamp>${maven.build.timestamp}</timestamp>
|
<timestamp>${maven.build.timestamp}</timestamp>
|
||||||
<maven.build.timestamp.format>E MMM dd HH:mm:ss z yyyy</maven.build.timestamp.format>
|
<maven.build.timestamp.format>E MMM dd HH:mm:ss z yyyy</maven.build.timestamp.format>
|
||||||
|
<java.vesrion>11</java.vesrion>
|
||||||
</properties>
|
</properties>
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<dependency>
|
<dependency>
|
||||||
|
@ -40,6 +41,18 @@
|
||||||
<groupId>org.postgresql</groupId>
|
<groupId>org.postgresql</groupId>
|
||||||
<artifactId>postgresql</artifactId>
|
<artifactId>postgresql</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<!--Apache Camel -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.camel</groupId>
|
||||||
|
<artifactId>camel-core</artifactId>
|
||||||
|
<version>3.20.3</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.camel</groupId>
|
||||||
|
<artifactId>camel-oaipmh</artifactId>
|
||||||
|
<version>3.20.3</version>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
@ -47,6 +60,7 @@
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.springframework.boot</groupId>
|
<groupId>org.springframework.boot</groupId>
|
||||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||||
|
<version>1.5.8.RELEASE</version>
|
||||||
<configuration>
|
<configuration>
|
||||||
<mainClass>eu.dnetlib.validatorapi.ValidatorApiApplication</mainClass>
|
<mainClass>eu.dnetlib.validatorapi.ValidatorApiApplication</mainClass>
|
||||||
<executable>true</executable>
|
<executable>true</executable>
|
||||||
|
|
|
@ -0,0 +1,74 @@
|
||||||
|
package eu.dnetlib.validatorapi.controllers;
|
||||||
|
|
||||||
|
import eu.dnetlib.validatorapi.entities.SummaryResult;
|
||||||
|
import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
||||||
|
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
|
||||||
|
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
|
||||||
|
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@CrossOrigin(origins = "*")
|
||||||
|
@RequestMapping("/reports")
|
||||||
|
public class ReportController {
|
||||||
|
private final ValidationJobRepository validationJobRepository;
|
||||||
|
private final ValidationResultRepository validationResultRepository;
|
||||||
|
private final ValidationIssueRepository validationIssueRepository;
|
||||||
|
|
||||||
|
public ReportController(ValidationJobRepository validationJobRepository,
|
||||||
|
ValidationResultRepository validationResultRepository,
|
||||||
|
ValidationIssueRepository validationIssueRepository) {
|
||||||
|
this.validationJobRepository = validationJobRepository;
|
||||||
|
this.validationResultRepository = validationResultRepository;
|
||||||
|
this.validationIssueRepository = validationIssueRepository;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||||
|
public List<SummaryResult> getJobResults(@RequestParam(name = "jobId") int jobId){
|
||||||
|
return validationResultRepository.getSummaryResult(jobId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||||
|
public List<ValidationIssue> getRecordsByRule(@RequestParam(name = "jobId") int jobId,
|
||||||
|
@RequestParam(name = "ruleName") String ruleName){
|
||||||
|
return validationIssueRepository.getErrors(jobId, ruleName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||||
|
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId){
|
||||||
|
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId);
|
||||||
|
return extractRecordsGroupedByRule(resultList);
|
||||||
|
}
|
||||||
|
|
||||||
|
@RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||||
|
public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId,
|
||||||
|
@RequestParam(name = "ruleName") String ruleName){
|
||||||
|
List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId);
|
||||||
|
return extractRecordsGroupedByRule(resultList);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, List<String>> extractRecordsGroupedByRule(List<Object[]> resultList) {
|
||||||
|
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
|
||||||
|
|
||||||
|
for (Object[] result : resultList) {
|
||||||
|
String issueText = (String) result[0];
|
||||||
|
String recordUrl = (String) result[1];
|
||||||
|
|
||||||
|
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
|
||||||
|
if (recordUrls == null) {
|
||||||
|
recordUrls = new ArrayList<>();
|
||||||
|
recordUrlsByIssueText.put(issueText, recordUrls);
|
||||||
|
}
|
||||||
|
recordUrls.add(recordUrl);
|
||||||
|
}
|
||||||
|
return recordUrlsByIssueText;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -3,14 +3,20 @@ package eu.dnetlib.validatorapi.controllers;
|
||||||
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
||||||
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
||||||
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
||||||
import eu.dnetlib.validatorapi.entities.*;
|
import eu.dnetlib.validatorapi.entities.RuleInfo;
|
||||||
|
import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
||||||
|
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||||
|
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
|
||||||
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
|
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
|
||||||
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
|
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
|
||||||
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
|
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
|
||||||
|
import eu.dnetlib.validatorapi.routes.OaiPmhRoute;
|
||||||
|
import org.apache.camel.CamelContext;
|
||||||
|
import org.apache.camel.builder.RouteBuilder;
|
||||||
|
import org.apache.camel.impl.DefaultCamelContext;
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.http.MediaType;
|
|
||||||
import org.springframework.web.bind.annotation.*;
|
import org.springframework.web.bind.annotation.*;
|
||||||
import org.w3c.dom.Document;
|
import org.w3c.dom.Document;
|
||||||
import org.w3c.dom.Element;
|
import org.w3c.dom.Element;
|
||||||
|
@ -34,7 +40,10 @@ import java.io.StringReader;
|
||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
import java.net.HttpURLConnection;
|
import java.net.HttpURLConnection;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.*;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@CrossOrigin(origins = "*")
|
@CrossOrigin(origins = "*")
|
||||||
|
@ -46,13 +55,15 @@ public class ValidationController {
|
||||||
private final ValidationIssueRepository validationIssueRepository;
|
private final ValidationIssueRepository validationIssueRepository;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
public ValidationController(ValidationJobRepository validationJobRepository, ValidationResultRepository validationResultRepository, ValidationIssueRepository validationIssueRepository) {
|
public ValidationController(ValidationJobRepository validationJobRepository,
|
||||||
|
ValidationResultRepository validationResultRepository,
|
||||||
|
ValidationIssueRepository validationIssueRepository) {
|
||||||
this.validationJobRepository = validationJobRepository;
|
this.validationJobRepository = validationJobRepository;
|
||||||
this.validationResultRepository = validationResultRepository;
|
this.validationResultRepository = validationResultRepository;
|
||||||
this.validationIssueRepository = validationIssueRepository;
|
this.validationIssueRepository = validationIssueRepository;
|
||||||
}
|
}
|
||||||
|
|
||||||
@RequestMapping(value = {"/realValidator"}, method = RequestMethod.POST)
|
@RequestMapping(value = {"/realValidator-old"}, method = RequestMethod.GET)
|
||||||
public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
||||||
@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now
|
@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now
|
||||||
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
|
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
|
||||||
|
@ -74,7 +85,6 @@ public class ValidationController {
|
||||||
double resultSum = 0;
|
double resultSum = 0;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
System.out.println("making request....");
|
|
||||||
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||||
DocumentBuilder db = dbf.newDocumentBuilder();
|
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||||
|
|
||||||
|
@ -82,46 +92,19 @@ public class ValidationController {
|
||||||
if (set != null) {
|
if (set != null) {
|
||||||
url += "&set=" + set;
|
url += "&set=" + set;
|
||||||
}
|
}
|
||||||
System.out.println("URL " + url.toString());
|
|
||||||
|
|
||||||
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||||
Document oaipmhResponse = parseResponse(conn.getInputStream());
|
Document oaipmhResponse = parseResponse(conn.getInputStream());
|
||||||
List<String> records = new ArrayList<String>();
|
|
||||||
|
|
||||||
//TODO: follow other approach if records are more than 100!!!
|
//TODO: follow other approach if records are more than 100!!!
|
||||||
int recordsRetrieved = 0;
|
int recordsRetrieved = 0;
|
||||||
// Step 2: Retrieve additional records using resumptionToken
|
List<String> records = new ArrayList<String>();
|
||||||
while (recordsRetrieved < batchSize) {
|
// Retrieve additional records using resumptionToken
|
||||||
System.out.println("True");
|
getNumberOfRecords(baseURL, batchSize, oaipmhResponse, records, recordsRetrieved);
|
||||||
Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0);
|
|
||||||
if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) {
|
|
||||||
System.out.println("breaking");
|
|
||||||
break; // no more records to retrieve
|
|
||||||
}
|
|
||||||
url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent();
|
|
||||||
conn = (HttpURLConnection) new URL(url).openConnection();
|
|
||||||
oaipmhResponse = parseResponse(conn.getInputStream());
|
|
||||||
|
|
||||||
XPathFactory xfactory = XPathFactory.newInstance();
|
|
||||||
XPath xpath = xfactory.newXPath();
|
|
||||||
XPathExpression recordsExpression = xpath.compile("//record");
|
|
||||||
NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET);
|
|
||||||
|
|
||||||
for (int i = 0; i < recordNodes.getLength(); ++i) {
|
|
||||||
Node element = recordNodes.item(i);
|
|
||||||
StringWriter stringWriter = new StringWriter();
|
|
||||||
Transformer xform = TransformerFactory.newInstance().newTransformer();
|
|
||||||
xform.transform(new DOMSource(element), new StreamResult(stringWriter));
|
|
||||||
records.add(stringWriter.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
recordsRetrieved += records.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (String recordXml : records) {
|
for (String recordXml : records) {
|
||||||
System.out.println("\n\n\n\n\n\n\n\n HELLLOOOOOOOOOO \n\n\n\n\n");
|
|
||||||
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
|
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
|
||||||
System.out.println("\n\n\n\n RECORD : " + recordXml + "\n\n\n\n\n");
|
|
||||||
|
|
||||||
if (profile != null) {
|
if (profile != null) {
|
||||||
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
|
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
|
||||||
|
@ -130,13 +113,9 @@ public class ValidationController {
|
||||||
for (Map.Entry entry : results.entrySet()) {
|
for (Map.Entry entry : results.entrySet()) {
|
||||||
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||||
String recordUrl = extractRecordUrl(doc, "identifier");
|
String recordUrl = extractRecordUrl(doc, "identifier");
|
||||||
System.out.println("DC IDENTIFIER: " + recordUrl);
|
|
||||||
String ruleName = entry.getKey().toString();
|
String ruleName = entry.getKey().toString();
|
||||||
ValidationRuleResult validationRuleResult = constructValidationRuleResult(validationJob.id, recordUrl,
|
ValidationRuleResult validationRuleResult = constructValidationRuleResult(validationJob.id, recordUrl,
|
||||||
ruleName, profile, engineResult);
|
ruleName, profile, engineResult);
|
||||||
|
|
||||||
System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
|
|
||||||
|
|
||||||
resultSum += engineResult.score();
|
resultSum += engineResult.score();
|
||||||
|
|
||||||
validationResultRepository.save(validationRuleResult);
|
validationResultRepository.save(validationRuleResult);
|
||||||
|
@ -144,20 +123,19 @@ public class ValidationController {
|
||||||
}
|
}
|
||||||
|
|
||||||
record++;
|
record++;
|
||||||
System.out.println(record);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
validationJob.status = "COMPLETED";
|
validationJob.status = "COMPLETED";
|
||||||
|
|
||||||
}
|
}
|
||||||
catch (Exception e) {
|
catch (Exception e) {
|
||||||
log.error("Validation job stopped unexpectedly.", e.getMessage());
|
log.error("Validation job stopped unexpectedly." + e.getMessage());
|
||||||
System.out.println("ERROR " + e.getMessage());
|
System.out.println("ERROR " + e.getMessage());
|
||||||
validationJob.status = "STOPPED";
|
validationJob.status = "STOPPED";
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
validationJob.endDate = new Date();
|
validationJob.endDate = new Date();
|
||||||
System.out.println("Final validation job "+ validationJob.hashCode());
|
//System.out.println("Final validation job "+ validationJob.hashCode());
|
||||||
validationJob.recordsTested = record;
|
validationJob.recordsTested = record;
|
||||||
validationJob.score = resultSum / record;
|
validationJob.score = resultSum / record;
|
||||||
//TODO uncomment
|
//TODO uncomment
|
||||||
|
@ -168,6 +146,85 @@ public class ValidationController {
|
||||||
//xmlValidationResponse.setFairRules(fairRules);
|
//xmlValidationResponse.setFairRules(fairRules);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void getNumberOfRecords(String baseURL, int batchSize, Document oaipmhResponse, List<String> records, int recordsRetrieved) throws Exception {
|
||||||
|
String url;
|
||||||
|
HttpURLConnection conn;
|
||||||
|
while (recordsRetrieved < batchSize) {
|
||||||
|
Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0);
|
||||||
|
if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) {
|
||||||
|
break; // no more records to retrieve
|
||||||
|
}
|
||||||
|
url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent();
|
||||||
|
conn = (HttpURLConnection) new URL(url).openConnection();
|
||||||
|
oaipmhResponse = parseResponse(conn.getInputStream());
|
||||||
|
|
||||||
|
XPathFactory xfactory = XPathFactory.newInstance();
|
||||||
|
XPath xpath = xfactory.newXPath();
|
||||||
|
XPathExpression recordsExpression = xpath.compile("//record");
|
||||||
|
NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET);
|
||||||
|
|
||||||
|
for (int i = 0; i < recordNodes.getLength(); ++i) {
|
||||||
|
Node element = recordNodes.item(i);
|
||||||
|
StringWriter stringWriter = new StringWriter();
|
||||||
|
Transformer xform = TransformerFactory.newInstance().newTransformer();
|
||||||
|
xform.transform(new DOMSource(element), new StreamResult(stringWriter));
|
||||||
|
records.add(stringWriter.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
recordsRetrieved += records.size();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@RequestMapping(value = {"/realValidator"}, method = RequestMethod.GET)
|
||||||
|
public void validateWithApacheCamel(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
||||||
|
@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now
|
||||||
|
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
|
||||||
|
@RequestParam(name="set", required = false) String set, @RequestParam(name="metadataPrefix", defaultValue = "oai_dc") String metadataPrefix) {
|
||||||
|
|
||||||
|
AbstractOpenAireProfile profile = initializeOpenAireProfile(guidelinesProfileName);
|
||||||
|
AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName);
|
||||||
|
|
||||||
|
if (profile == null && fairProfile == null) {
|
||||||
|
log.error("Exception: No valid guidelines " + guidelinesProfileName + ".");
|
||||||
|
new Exception("Validation Job stopped unexpectedly. No valid guidelines " +
|
||||||
|
guidelinesProfileName + " were provided.");
|
||||||
|
}
|
||||||
|
|
||||||
|
ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords);
|
||||||
|
validationJob.guidelines = profile.name();
|
||||||
|
validationJobRepository.save(validationJob);
|
||||||
|
|
||||||
|
int record = 0;
|
||||||
|
double resultSum = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||||
|
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||||
|
CamelContext camelContext = new DefaultCamelContext();
|
||||||
|
RouteBuilder oaiPmhRouteBuilder =
|
||||||
|
new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
|
||||||
|
profile, validationJob, numberOfRecords, validationIssueRepository, validationResultRepository);
|
||||||
|
camelContext.addRoutes(oaiPmhRouteBuilder);
|
||||||
|
camelContext.start();
|
||||||
|
//camelContext.stop();
|
||||||
|
validationJob.status = "COMPLETED";
|
||||||
|
|
||||||
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
log.error("Validation job stopped unexpectedly." + e.getMessage());
|
||||||
|
System.out.println("ERROR " + e.getMessage());
|
||||||
|
validationJob.status = "STOPPED";
|
||||||
|
|
||||||
|
} finally {
|
||||||
|
validationJob.endDate = new Date();
|
||||||
|
System.out.println("Final validation job "+ validationJob.hashCode());
|
||||||
|
validationJob.recordsTested = record;
|
||||||
|
validationJob.score = resultSum / record;
|
||||||
|
validationJobRepository.save(validationJob);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
private void constructValidationJobResult(ValidationJob validationJob, ValidationRuleResult validationRuleResult,
|
private void constructValidationJobResult(ValidationJob validationJob, ValidationRuleResult validationRuleResult,
|
||||||
Map.Entry entry, AbstractOpenAireProfile profile, Document document) {
|
Map.Entry entry, AbstractOpenAireProfile profile, Document document) {
|
||||||
|
@ -199,9 +256,8 @@ public class ValidationController {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
|
private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
|
||||||
System.out.println("-------");
|
|
||||||
for (String error:engineResult.errors()) {
|
for (String error:engineResult.errors()) {
|
||||||
System.out.println("11111");
|
/*System.out.println("11111");*/
|
||||||
ValidationIssue validationIssue = new ValidationIssue();
|
ValidationIssue validationIssue = new ValidationIssue();
|
||||||
validationIssue.validationJobId = validationJobId;
|
validationIssue.validationJobId = validationJobId;
|
||||||
validationIssue.ruleName = ruleName;
|
validationIssue.ruleName = ruleName;
|
||||||
|
@ -209,11 +265,15 @@ public class ValidationController {
|
||||||
validationIssue.issueType = "ERROR";
|
validationIssue.issueType = "ERROR";
|
||||||
validationIssue.issueText = error;
|
validationIssue.issueText = error;
|
||||||
validationIssueRepository.save(validationIssue);
|
validationIssueRepository.save(validationIssue);
|
||||||
|
/*
|
||||||
System.out.println(validationIssue);
|
System.out.println(validationIssue);
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
for (String warning: engineResult.warnings()){
|
for (String warning: engineResult.warnings()){
|
||||||
|
/*
|
||||||
System.out.println("22222");
|
System.out.println("22222");
|
||||||
|
*/
|
||||||
ValidationIssue validationIssue = new ValidationIssue();
|
ValidationIssue validationIssue = new ValidationIssue();
|
||||||
validationIssue.validationJobId = validationJobId;
|
validationIssue.validationJobId = validationJobId;
|
||||||
validationIssue.ruleName = ruleName;
|
validationIssue.ruleName = ruleName;
|
||||||
|
@ -221,7 +281,9 @@ public class ValidationController {
|
||||||
validationIssue.issueType = "WARNING";
|
validationIssue.issueType = "WARNING";
|
||||||
validationIssue.issueText = warning;
|
validationIssue.issueText = warning;
|
||||||
validationIssueRepository.save(validationIssue);
|
validationIssueRepository.save(validationIssue);
|
||||||
|
/*
|
||||||
System.out.println(validationIssue);
|
System.out.println(validationIssue);
|
||||||
|
*/
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -360,7 +422,7 @@ public class ValidationController {
|
||||||
|
|
||||||
}
|
}
|
||||||
catch (Exception e) {
|
catch (Exception e) {
|
||||||
log.error("Validation job stopped unexpectedly.", e.getMessage());
|
log.error("Validation job stopped unexpectedly." + e.getMessage());
|
||||||
System.out.println("ERROR " + e.getMessage());
|
System.out.println("ERROR " + e.getMessage());
|
||||||
validationJob.status = "STOPPED";
|
validationJob.status = "STOPPED";
|
||||||
|
|
||||||
|
@ -376,90 +438,7 @@ public class ValidationController {
|
||||||
//xmlValidationResponse.setFairRules(fairRules);
|
//xmlValidationResponse.setFairRules(fairRules);
|
||||||
}
|
}
|
||||||
|
|
||||||
@RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.GET,
|
|
||||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
|
||||||
public List<SummaryResult> getJobResults(@RequestParam(name = "jobId") int jobId){
|
|
||||||
System.out.println(validationResultRepository.getSummaryResult(jobId).toString());
|
|
||||||
System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult(jobId).get(0));
|
|
||||||
return validationResultRepository.getSummaryResult(jobId);
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
@RequestMapping(value = {"getErrors"}, method = RequestMethod.GET,
|
|
||||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
|
||||||
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
|
|
||||||
|
|
||||||
List<Object[]> resultList = validationIssueRepository.getErrorsRecordUrls(jobId, ruleName);
|
|
||||||
|
|
||||||
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
|
|
||||||
|
|
||||||
for (Object[] result : resultList) {
|
|
||||||
String issueText = (String) result[0];
|
|
||||||
String recordUrl = (String) result[1];
|
|
||||||
|
|
||||||
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
|
|
||||||
if (recordUrls == null) {
|
|
||||||
recordUrls = new ArrayList<>();
|
|
||||||
recordUrlsByIssueText.put(issueText, recordUrls);
|
|
||||||
}
|
|
||||||
recordUrls.add(recordUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println("issues " + recordUrlsByIssueText.keySet());
|
|
||||||
|
|
||||||
return recordUrlsByIssueText;
|
|
||||||
}*/
|
|
||||||
|
|
||||||
@RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET,
|
|
||||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
|
||||||
public List<ValidationIssue> getRecordsByRule(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
|
|
||||||
return validationIssueRepository.getErrors(jobId, ruleName);
|
|
||||||
}
|
|
||||||
|
|
||||||
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET,
|
|
||||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
|
||||||
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId){
|
|
||||||
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId);
|
|
||||||
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
|
|
||||||
|
|
||||||
for (Object[] result : resultList) {
|
|
||||||
String issueText = (String) result[0];
|
|
||||||
String recordUrl = (String) result[1];
|
|
||||||
|
|
||||||
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
|
|
||||||
if (recordUrls == null) {
|
|
||||||
recordUrls = new ArrayList<>();
|
|
||||||
recordUrlsByIssueText.put(issueText, recordUrls);
|
|
||||||
}
|
|
||||||
recordUrls.add(recordUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println("issues " + recordUrlsByIssueText.keySet());
|
|
||||||
|
|
||||||
return recordUrlsByIssueText;
|
|
||||||
}
|
|
||||||
|
|
||||||
@RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET,
|
|
||||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
|
||||||
public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
|
|
||||||
List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId);
|
|
||||||
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
|
|
||||||
|
|
||||||
for (Object[] result : resultList) {
|
|
||||||
String issueText = (String) result[0];
|
|
||||||
String recordUrl = (String) result[1];
|
|
||||||
|
|
||||||
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
|
|
||||||
if (recordUrls == null) {
|
|
||||||
recordUrls = new ArrayList<>();
|
|
||||||
recordUrlsByIssueText.put(issueText, recordUrls);
|
|
||||||
}
|
|
||||||
recordUrls.add(recordUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println("issues " + recordUrlsByIssueText.keySet());
|
|
||||||
|
|
||||||
return recordUrlsByIssueText;
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<String> extractRecordXmls(String xml) throws Exception {
|
public List<String> extractRecordXmls(String xml) throws Exception {
|
||||||
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
|
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
|
||||||
|
|
|
@ -2,7 +2,6 @@ package eu.dnetlib.validatorapi.controllers;
|
||||||
|
|
||||||
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
||||||
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
||||||
import eu.dnetlib.validator2.validation.guideline.StandardResult;
|
|
||||||
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
||||||
import eu.dnetlib.validatorapi.entities.RuleInfo;
|
import eu.dnetlib.validatorapi.entities.RuleInfo;
|
||||||
import eu.dnetlib.validatorapi.entities.XmlValidationResponse;
|
import eu.dnetlib.validatorapi.entities.XmlValidationResponse;
|
||||||
|
@ -15,7 +14,9 @@ import org.xml.sax.InputSource;
|
||||||
|
|
||||||
import javax.xml.parsers.DocumentBuilder;
|
import javax.xml.parsers.DocumentBuilder;
|
||||||
import javax.xml.parsers.DocumentBuilderFactory;
|
import javax.xml.parsers.DocumentBuilderFactory;
|
||||||
import java.io.*;
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.StringReader;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
|
@ -1,7 +1,5 @@
|
||||||
package eu.dnetlib.validatorapi.entities;
|
package eu.dnetlib.validatorapi.entities;
|
||||||
|
|
||||||
import org.springframework.boot.autoconfigure.session.StoreType;
|
|
||||||
|
|
||||||
import javax.persistence.Entity;
|
import javax.persistence.Entity;
|
||||||
import javax.persistence.Id;
|
import javax.persistence.Id;
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,186 @@
|
||||||
|
package eu.dnetlib.validatorapi.processors;
|
||||||
|
|
||||||
|
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
||||||
|
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
||||||
|
import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
|
||||||
|
import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
||||||
|
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||||
|
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
|
||||||
|
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
|
||||||
|
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
|
||||||
|
import org.apache.camel.Exchange;
|
||||||
|
import org.apache.camel.Processor;
|
||||||
|
import org.w3c.dom.Document;
|
||||||
|
import org.w3c.dom.Element;
|
||||||
|
import org.w3c.dom.NodeList;
|
||||||
|
import org.xml.sax.InputSource;
|
||||||
|
|
||||||
|
import javax.xml.parsers.DocumentBuilder;
|
||||||
|
import javax.xml.parsers.DocumentBuilderFactory;
|
||||||
|
import java.io.StringReader;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public class XmlProcessor implements Processor {
|
||||||
|
private final AbstractOpenAireProfile profile;
|
||||||
|
private final ValidationJob validationJob;
|
||||||
|
private long processedRecords;
|
||||||
|
|
||||||
|
private final ValidationIssueRepository validationIssueRepository;
|
||||||
|
private final ValidationResultRepository validationResultRepository;
|
||||||
|
|
||||||
|
public XmlProcessor(final AbstractOpenAireProfile profile, final ValidationJob validationJob,
|
||||||
|
final ValidationIssueRepository validationIssueRepository,
|
||||||
|
final ValidationResultRepository validationResultRepository){
|
||||||
|
super();
|
||||||
|
this.profile = profile;
|
||||||
|
this.validationJob = validationJob;
|
||||||
|
this.validationIssueRepository = validationIssueRepository;
|
||||||
|
this.validationResultRepository = validationResultRepository;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void process(Exchange exchange) throws Exception {
|
||||||
|
String recordXml = exchange.getIn().getBody(String.class);
|
||||||
|
|
||||||
|
final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||||
|
final DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
|
||||||
|
Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml)));
|
||||||
|
String recordUrl = extractRecordUrl(doc, "identifier");
|
||||||
|
|
||||||
|
processedRecords++;
|
||||||
|
System.out.println("processed " + processedRecords);
|
||||||
|
exchange.setProperty("totalRecords", processedRecords);
|
||||||
|
|
||||||
|
if (profile != null) {
|
||||||
|
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
|
||||||
|
System.out.println("DC IDENTIFIER: " + recordUrl);
|
||||||
|
|
||||||
|
Map<String, Guideline.Result> results = validationResult.results();
|
||||||
|
|
||||||
|
ValidationRuleResult validationRuleResult;
|
||||||
|
for (Map.Entry entry : results.entrySet()) {
|
||||||
|
final Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||||
|
String ruleName = entry.getKey().toString();
|
||||||
|
validationRuleResult = new ValidationRuleResult();
|
||||||
|
constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
|
||||||
|
ruleName, profile, engineResult);
|
||||||
|
validationResultRepository.save(validationRuleResult);
|
||||||
|
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult);
|
||||||
|
|
||||||
|
// System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
|
||||||
|
for (String error:engineResult.errors()) {
|
||||||
|
/*System.out.println("11111");*/
|
||||||
|
ValidationIssue validationIssue = new ValidationIssue();
|
||||||
|
validationIssue.validationJobId = validationJobId;
|
||||||
|
validationIssue.ruleName = ruleName;
|
||||||
|
validationIssue.recordUrl = recordUrl;
|
||||||
|
validationIssue.issueType = "ERROR";
|
||||||
|
validationIssue.issueText = error;
|
||||||
|
validationIssueRepository.save(validationIssue);
|
||||||
|
/*
|
||||||
|
System.out.println(validationIssue);
|
||||||
|
*/
|
||||||
|
}
|
||||||
|
|
||||||
|
for (String warning: engineResult.warnings()){
|
||||||
|
/*
|
||||||
|
System.out.println("22222");
|
||||||
|
*/
|
||||||
|
ValidationIssue validationIssue = new ValidationIssue();
|
||||||
|
validationIssue.validationJobId = validationJobId;
|
||||||
|
validationIssue.ruleName = ruleName;
|
||||||
|
validationIssue.recordUrl = recordUrl;
|
||||||
|
validationIssue.issueType = "WARNING";
|
||||||
|
validationIssue.issueText = warning;
|
||||||
|
validationIssueRepository.save(validationIssue);
|
||||||
|
/*
|
||||||
|
System.out.println(validationIssue);
|
||||||
|
*/
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/*public void process2(Exchange exchange) throws Exception {
|
||||||
|
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||||
|
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||||
|
String recordXml = exchange.getIn().getBody(String.class);
|
||||||
|
|
||||||
|
// System.out.println("\n\n\n\n\n\n\n\n\n\n\n\n");
|
||||||
|
System.out.println(recordXml);
|
||||||
|
// System.out.println("\n\n\n\n\n\n\n\n\n\n\n\n");
|
||||||
|
|
||||||
|
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
|
||||||
|
|
||||||
|
|
||||||
|
if (profile != null) {
|
||||||
|
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
|
||||||
|
String recordUrl = extractRecordUrl(doc, "identifier");
|
||||||
|
System.out.println("DC IDENTIFIER: " + recordUrl);
|
||||||
|
|
||||||
|
Map<String, Guideline.Result> results = validationResult.results();
|
||||||
|
for (Map.Entry entry : results.entrySet()) {
|
||||||
|
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||||
|
// System.out.println("engine result > " + engineResult);
|
||||||
|
|
||||||
|
String ruleName = entry.getKey().toString();
|
||||||
|
// System.out.println("rulename: " + recordUrl);
|
||||||
|
|
||||||
|
ValidationRuleResult validationRuleResult = new ValidationRuleResult();
|
||||||
|
// System.out.println("validationjob.id " + validationJob.id);
|
||||||
|
// System.out.println("recordUrl " + recordUrl);
|
||||||
|
// System.out.println("rulename " + ruleName);
|
||||||
|
// System.out.println("profile " + profile);
|
||||||
|
// System.out.println("engineResult " + engineResult);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
|
||||||
|
ruleName, profile, engineResult);
|
||||||
|
|
||||||
|
// System.out.println("xaxaxa");
|
||||||
|
// System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
|
||||||
|
// resultSum += engineResult.score();
|
||||||
|
|
||||||
|
// elements = nodeListToList(recordNodes); // assign the Elements to the variable
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
processedRecords++;
|
||||||
|
System.out.println("processed " + processedRecords);
|
||||||
|
exchange.setProperty("totalRecords", processedRecords);
|
||||||
|
}*/
|
||||||
|
|
||||||
|
//TODO consider throwing exception - UTIL class?
|
||||||
|
private String extractRecordUrl(final Document document, final String xmlField) {
|
||||||
|
NodeList identifierNodes = document.getElementsByTagName(xmlField);
|
||||||
|
if (identifierNodes.getLength() > 0) {
|
||||||
|
Element identifierElement = (Element) identifierNodes.item(0);
|
||||||
|
return identifierElement.getTextContent();
|
||||||
|
}
|
||||||
|
return "-";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String recordUrl, String ruleName,
|
||||||
|
AbstractOpenAireProfile profile, Guideline.Result engineResult) {
|
||||||
|
|
||||||
|
validationRuleResult.validationJobId = validationJobId;
|
||||||
|
validationRuleResult.ruleName = ruleName;
|
||||||
|
validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight();
|
||||||
|
validationRuleResult.recordUrl = recordUrl;
|
||||||
|
validationRuleResult.score = engineResult.score();
|
||||||
|
validationRuleResult.status = engineResult.status().toString();
|
||||||
|
validationRuleResult.internalError = engineResult.internalError();
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,18 +1,12 @@
|
||||||
package eu.dnetlib.validatorapi.repositories;
|
package eu.dnetlib.validatorapi.repositories;
|
||||||
|
|
||||||
import eu.dnetlib.validatorapi.entities.SummaryResult;
|
|
||||||
import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
import org.springframework.data.jpa.repository.Query;
|
import org.springframework.data.jpa.repository.Query;
|
||||||
import org.springframework.data.repository.query.Param;
|
import org.springframework.data.repository.query.Param;
|
||||||
import org.springframework.stereotype.Repository;
|
import org.springframework.stereotype.Repository;
|
||||||
|
|
||||||
import javax.persistence.EntityManager;
|
|
||||||
import javax.persistence.PersistenceContext;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
@Repository
|
@Repository
|
||||||
public interface ValidationIssueRepository extends JpaRepository<ValidationIssue, Long> {
|
public interface ValidationIssueRepository extends JpaRepository<ValidationIssue, Long> {
|
||||||
|
@ -30,7 +24,6 @@ public interface ValidationIssueRepository extends JpaRepository<ValidationIssue
|
||||||
"GROUP BY vi.issueText, vi.recordUrl")
|
"GROUP BY vi.issueText, vi.recordUrl")
|
||||||
List<ValidationIssue> getErrors(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
|
List<ValidationIssue> getErrors(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
|
||||||
|
|
||||||
|
|
||||||
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
|
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
|
||||||
"FROM ValidationIssue vi " +
|
"FROM ValidationIssue vi " +
|
||||||
"WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" +
|
"WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" +
|
||||||
|
|
|
@ -0,0 +1,130 @@
|
||||||
|
package eu.dnetlib.validatorapi.routes;
|
||||||
|
|
||||||
|
import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
|
||||||
|
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||||
|
import eu.dnetlib.validatorapi.processors.XmlProcessor;
|
||||||
|
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
|
||||||
|
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
|
||||||
|
import org.apache.camel.Exchange;
|
||||||
|
import org.apache.camel.Processor;
|
||||||
|
import org.apache.camel.builder.RouteBuilder;
|
||||||
|
|
||||||
|
public class OaiPmhRoute extends RouteBuilder {
|
||||||
|
|
||||||
|
private String oaiEndpoint = "oaipmh://http://repositorium.sdum.uminho.pt/oai/request";
|
||||||
|
|
||||||
|
private AbstractOpenAireProfile profile;
|
||||||
|
private long maxNumberOfRecords = 0;
|
||||||
|
private ValidationJob validationJob;
|
||||||
|
private final ValidationIssueRepository validationIssueRepository;
|
||||||
|
private final ValidationResultRepository validationResultRepository;
|
||||||
|
|
||||||
|
public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob,
|
||||||
|
long maxNumberOfRecords, final ValidationIssueRepository validationIssueRepository,
|
||||||
|
final ValidationResultRepository validationResultRepository) {
|
||||||
|
this.oaiEndpoint = oaiEndpoint;
|
||||||
|
this.validationJob = validationJob;
|
||||||
|
this.profile = profile;
|
||||||
|
this.maxNumberOfRecords = maxNumberOfRecords;
|
||||||
|
this.validationIssueRepository = validationIssueRepository;
|
||||||
|
this.validationResultRepository = validationResultRepository;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void configure() throws Exception {
|
||||||
|
|
||||||
|
|
||||||
|
from(oaiEndpoint)
|
||||||
|
.setProperty("totalRecords", constant(0))
|
||||||
|
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 10")
|
||||||
|
.split(xpath("//*[local-name()='record']"))
|
||||||
|
// .log("\n\n\n----------------\n\n\n\n\n\n${body}\n\n\n----------------\n\n\n\n\n\n")
|
||||||
|
.process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository))
|
||||||
|
.end().to("direct:end");
|
||||||
|
|
||||||
|
/* from("oaipmh://http://repositorium.sdum.uminho.pt/oai/request?verb=ListRecords&metadataPrefix=oai_dc")
|
||||||
|
.setProperty("totalRecords", constant(0))
|
||||||
|
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 10")
|
||||||
|
.split(xpath("//*[local-name()='record']"))
|
||||||
|
//.log("\n\n\n----------------\n\n\n\n\n\n${body}\n\n\n----------------\n\n\n\n\n\n")
|
||||||
|
.process(new XmlProcessor(profile, validationJob))
|
||||||
|
.end().to("direct:end");*/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*** kai auto from(oaiEndpoint) // trigger the route every minute
|
||||||
|
.split(xpath("//*[local-name()='record']"))
|
||||||
|
.setProperty("totalRecords", constant(0))
|
||||||
|
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 50")
|
||||||
|
.process(new XmlProcessor(profile, validationJob))
|
||||||
|
.end()
|
||||||
|
.to("direct:end");
|
||||||
|
**/
|
||||||
|
//from(oaiEndpoint).process(new XmlProcessor(profile,validationJob));
|
||||||
|
|
||||||
|
// from("direct:oaiRequest").process(new DummyXMLProcessor());
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*** AUTO PAIZEI !!!!!! ***/
|
||||||
|
/* from(oaiEndpoint).
|
||||||
|
split(xpath("//*[local-name()='record']")).process(new XmlProcessor(profile, validationJob));
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*** KAI AUTO !!!!!! ***/
|
||||||
|
/*
|
||||||
|
from("timer://myTimer?fixedRate=true&period=60000") // trigger the route every minute
|
||||||
|
.setProperty("totalRecords", constant(0))
|
||||||
|
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 50")
|
||||||
|
.to("direct:oaiRequest")
|
||||||
|
.process(new RecordCountProcessor())
|
||||||
|
.end()
|
||||||
|
.to("direct:end");
|
||||||
|
|
||||||
|
from("direct:oaiRequest").
|
||||||
|
process(exchange -> {
|
||||||
|
int counter = exchange.getProperty("totalRecords", 0, Integer.class);
|
||||||
|
System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class));
|
||||||
|
counter++;
|
||||||
|
System.out.println("counter ++" + counter);
|
||||||
|
exchange.setProperty("processedRecords", counter);
|
||||||
|
System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class));
|
||||||
|
|
||||||
|
});
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** KAI AUTO ***/
|
||||||
|
/*
|
||||||
|
from("timer://myTimer?fixedRate=true&period=60000") // trigger the route every minute
|
||||||
|
.setProperty("totalRecords", constant(0))
|
||||||
|
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 50")
|
||||||
|
.to("direct:oaiRequest")
|
||||||
|
.process(new RecordCountProcessor())
|
||||||
|
.end()
|
||||||
|
.to("direct:end");
|
||||||
|
|
||||||
|
from("direct:oaiRequest").
|
||||||
|
process(exchange -> {
|
||||||
|
int counter = exchange.getProperty("totalRecords", 0, Integer.class);
|
||||||
|
System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class));
|
||||||
|
counter++;
|
||||||
|
System.out.println("counter ++" + counter);
|
||||||
|
exchange.setProperty("processedRecords", counter);
|
||||||
|
System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class));
|
||||||
|
|
||||||
|
});
|
||||||
|
*/
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class RecordCountProcessor implements Processor {
|
||||||
|
@Override
|
||||||
|
public void process(Exchange exchange) throws Exception {
|
||||||
|
System.out.println("RecordCountProcessor");
|
||||||
|
|
||||||
|
int processedRecords = exchange.getProperty("processedRecords", 0, Integer.class);
|
||||||
|
exchange.setProperty("totalRecords", processedRecords);
|
||||||
|
|
||||||
|
}
|
||||||
|
}}
|
|
@ -1,33 +0,0 @@
|
||||||
package eu.dnetlib.validatorapi.utils;
|
|
||||||
|
|
||||||
public class XmlProcessor { /* implements Processor {
|
|
||||||
private List<Element> elements;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void process(Exchange exchange) throws Exception {
|
|
||||||
String xml = exchange.getIn().getBody(String.class);
|
|
||||||
|
|
||||||
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
|
|
||||||
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
|
|
||||||
Document doc = dBuilder.parse(new InputSource(new StringReader(xml)));
|
|
||||||
|
|
||||||
XPathFactory xfactory = XPathFactory.newInstance();
|
|
||||||
XPath xpath = xfactory.newXPath();
|
|
||||||
XPathExpression recordsExpression = xpath.compile("//record");
|
|
||||||
NodeList recordNodes = (NodeList) recordsExpression.evaluate(doc, XPathConstants.NODESET);
|
|
||||||
|
|
||||||
elements = nodeListToList(recordNodes); // assign the Elements to the variable
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<Element> nodeListToList(NodeList nodeList) {
|
|
||||||
List<Element> list = new ArrayList<>();
|
|
||||||
for (int i = 0; i < nodeList.getLength(); i++) {
|
|
||||||
list.add((Element) nodeList.item(i));
|
|
||||||
}
|
|
||||||
return list;
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<Element> getElements() {
|
|
||||||
return elements;
|
|
||||||
}*/
|
|
||||||
}
|
|
|
@ -6,6 +6,9 @@
|
||||||
</Property>
|
</Property>
|
||||||
</Properties>
|
</Properties>
|
||||||
<Appenders>
|
<Appenders>
|
||||||
|
<Console name="console" target="SYSTEM_OUT" follow="true">
|
||||||
|
<PatternLayout pattern="${LOG_PATTERN}"/>
|
||||||
|
</Console>
|
||||||
<!-- Rolling File Appender -->
|
<!-- Rolling File Appender -->
|
||||||
<RollingFile name="R" fileName="/var/log/dnet/uoa-validator-api/uoa-validator-api.log"
|
<RollingFile name="R" fileName="/var/log/dnet/uoa-validator-api/uoa-validator-api.log"
|
||||||
filePattern="/var/log/dnet/uoa-validator-api/uoa-validator-api-%d{yyyy-MM-dd}-%i.log">
|
filePattern="/var/log/dnet/uoa-validator-api/uoa-validator-api-%d{yyyy-MM-dd}-%i.log">
|
||||||
|
@ -29,11 +32,17 @@
|
||||||
</RollingFile>
|
</RollingFile>
|
||||||
</Appenders>
|
</Appenders>
|
||||||
<Loggers>
|
<Loggers>
|
||||||
<Logger name="org.springframework" level="debug" additivity="false">
|
<Logger name="org.springframework" level="info" additivity="false">
|
||||||
<AppenderRef ref="S"/>
|
<AppenderRef ref="S"/>
|
||||||
|
<AppenderRef ref="console"/>
|
||||||
</Logger>
|
</Logger>
|
||||||
<Root level="debug">
|
<Logger name="org.springframework" level="info" additivity="false">
|
||||||
|
<AppenderRef ref="S"/>
|
||||||
|
<AppenderRef ref="console"/>
|
||||||
|
</Logger>
|
||||||
|
<Root level="info">
|
||||||
<AppenderRef ref="R"/>
|
<AppenderRef ref="R"/>
|
||||||
|
<AppenderRef ref="console"/>
|
||||||
</Root>
|
</Root>
|
||||||
</Loggers>
|
</Loggers>
|
||||||
</Configuration>
|
</Configuration>
|
|
@ -10598,7 +10598,8 @@
|
||||||
<setSpec>publications:year2017</setSpec>
|
<setSpec>publications:year2017</setSpec>
|
||||||
</header>
|
</header>
|
||||||
<metadata>
|
<metadata>
|
||||||
<oai_dc:dc xmlns="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:ns2="http://purl.org/dc/terms/">
|
<oai_dc:dc xmlns="http://purl.org/dc/elements/1.1/"
|
||||||
|
xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
|
||||||
<title xml:lang="eng">Report on challenges for SCIs:Smart Resilience Indicators for Smart Critical Infrastructures Deliverable D2.2</title>
|
<title xml:lang="eng">Report on challenges for SCIs:Smart Resilience Indicators for Smart Critical Infrastructures Deliverable D2.2</title>
|
||||||
<creator>Walther, G.</creator>
|
<creator>Walther, G.</creator>
|
||||||
<creator>Jovanovic, M.</creator>
|
<creator>Jovanovic, M.</creator>
|
||||||
|
|
Loading…
Reference in New Issue