MaDgIK
/
uoa-validator-api
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

progress with apache camel dynamic routes

This commit is contained in:
Katerina 2023-05-29 14:40:25 +03:00
parent a15670b741
commit fe7a8b1631
11 changed files with 529 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pom.xml
View File

@ -22,6 +22,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<timestamp>${maven.build.timestamp}</timestamp>
<maven.build.timestamp.format>E MMM dd HH:mm:ss z yyyy</maven.build.timestamp.format>
<java.vesrion>11</java.vesrion>
</properties>
<dependencies>
<dependency>
@ -40,6 +41,18 @@
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<!--Apache Camel -->
<dependency>
<groupId>org.apache.camel</groupId>
<artifactId>camel-core</artifactId>
<version>3.20.3</version>
</dependency>
<dependency>
<groupId>org.apache.camel</groupId>
<artifactId>camel-oaipmh</artifactId>
<version>3.20.3</version>
</dependency>
</dependencies>
<build>
@ -47,6 +60,7 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>1.5.8.RELEASE</version>
<configuration>
<mainClass>eu.dnetlib.validatorapi.ValidatorApiApplication</mainClass>
<executable>true</executable>

74
src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java Normal file
View File

@ -0,0 +1,74 @@
package eu.dnetlib.validatorapi.controllers;
import eu.dnetlib.validatorapi.entities.SummaryResult;
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@RestController
@CrossOrigin(origins = "*")
@RequestMapping("/reports")
public class ReportController {
private final ValidationJobRepository validationJobRepository;
private final ValidationResultRepository validationResultRepository;
private final ValidationIssueRepository validationIssueRepository;
public ReportController(ValidationJobRepository validationJobRepository,
ValidationResultRepository validationResultRepository,
ValidationIssueRepository validationIssueRepository) {
this.validationJobRepository = validationJobRepository;
this.validationResultRepository = validationResultRepository;
this.validationIssueRepository = validationIssueRepository;
}
@RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public List<SummaryResult> getJobResults(@RequestParam(name = "jobId") int jobId){
return validationResultRepository.getSummaryResult(jobId);
}
@RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public List<ValidationIssue> getRecordsByRule(@RequestParam(name = "jobId") int jobId,
@RequestParam(name = "ruleName") String ruleName){
return validationIssueRepository.getErrors(jobId, ruleName);
}
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId){
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId);
return extractRecordsGroupedByRule(resultList);
}
@RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId,
@RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId);
return extractRecordsGroupedByRule(resultList);
}
private Map<String, List<String>> extractRecordsGroupedByRule(List<Object[]> resultList) {
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
for (Object[] result : resultList) {
String issueText = (String) result[0];
String recordUrl = (String) result[1];
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
if (recordUrls == null) {
recordUrls = new ArrayList<>();
recordUrlsByIssueText.put(issueText, recordUrls);
}
recordUrls.add(recordUrl);
}
return recordUrlsByIssueText;
}
}

239
src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
View File

@ -3,14 +3,20 @@ package eu.dnetlib.validatorapi.controllers;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.openaire.*;
import eu.dnetlib.validatorapi.entities.*;
import eu.dnetlib.validatorapi.entities.RuleInfo;
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import eu.dnetlib.validatorapi.routes.OaiPmhRoute;
import org.apache.camel.CamelContext;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@ -34,7 +40,10 @@ import java.io.StringReader;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
@RestController
@CrossOrigin(origins = "*")
@ -46,13 +55,15 @@ public class ValidationController {
private final ValidationIssueRepository validationIssueRepository;
@Autowired
public ValidationController(ValidationJobRepository validationJobRepository, ValidationResultRepository validationResultRepository, ValidationIssueRepository validationIssueRepository) {
public ValidationController(ValidationJobRepository validationJobRepository,
ValidationResultRepository validationResultRepository,
ValidationIssueRepository validationIssueRepository) {
this.validationJobRepository = validationJobRepository;
this.validationResultRepository = validationResultRepository;
this.validationIssueRepository = validationIssueRepository;
}
@RequestMapping(value = {"/realValidator"}, method = RequestMethod.POST)
@RequestMapping(value = {"/realValidator-old"}, method = RequestMethod.GET)
public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
@ -74,7 +85,6 @@ public class ValidationController {
double resultSum = 0;
try {
System.out.println("making request....");
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
@ -82,46 +92,19 @@ public class ValidationController {
if (set != null) {
url += "&set=" + set;
}
System.out.println("URL " + url.toString());
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
Document oaipmhResponse = parseResponse(conn.getInputStream());
List<String> records = new ArrayList<String>();
//TODO: follow other approach if records are more than 100!!!
int recordsRetrieved = 0;
// Step 2: Retrieve additional records using resumptionToken
while (recordsRetrieved < batchSize) {
System.out.println("True");
Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0);
if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) {
System.out.println("breaking");
break; // no more records to retrieve
}
url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent();
conn = (HttpURLConnection) new URL(url).openConnection();
oaipmhResponse = parseResponse(conn.getInputStream());
XPathFactory xfactory = XPathFactory.newInstance();
XPath xpath = xfactory.newXPath();
XPathExpression recordsExpression = xpath.compile("//record");
NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET);
for (int i = 0; i < recordNodes.getLength(); ++i) {
Node element = recordNodes.item(i);
StringWriter stringWriter = new StringWriter();
Transformer xform = TransformerFactory.newInstance().newTransformer();
xform.transform(new DOMSource(element), new StreamResult(stringWriter));
records.add(stringWriter.toString());
}
recordsRetrieved += records.size();
}
List<String> records = new ArrayList<String>();
// Retrieve additional records using resumptionToken
getNumberOfRecords(baseURL, batchSize, oaipmhResponse, records, recordsRetrieved);
for (String recordXml : records) {
System.out.println("\n\n\n\n\n\n\n\n HELLLOOOOOOOOOO \n\n\n\n\n");
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
System.out.println("\n\n\n\n RECORD : " + recordXml + "\n\n\n\n\n");
if (profile != null) {
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
@ -130,13 +113,9 @@ public class ValidationController {
for (Map.Entry entry : results.entrySet()) {
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
String recordUrl = extractRecordUrl(doc, "identifier");
System.out.println("DC IDENTIFIER: " + recordUrl);
String ruleName = entry.getKey().toString();
ValidationRuleResult validationRuleResult = constructValidationRuleResult(validationJob.id, recordUrl,
ruleName, profile, engineResult);
System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
resultSum += engineResult.score();
validationResultRepository.save(validationRuleResult);
@ -144,20 +123,19 @@ public class ValidationController {
}
record++;
System.out.println(record);
}
}
validationJob.status = "COMPLETED";
}
catch (Exception e) {
log.error("Validation job stopped unexpectedly.", e.getMessage());
log.error("Validation job stopped unexpectedly." + e.getMessage());
System.out.println("ERROR " + e.getMessage());
validationJob.status = "STOPPED";
} finally {
validationJob.endDate = new Date();
System.out.println("Final validation job "+ validationJob.hashCode());
//System.out.println("Final validation job "+ validationJob.hashCode());
validationJob.recordsTested = record;
validationJob.score = resultSum / record;
//TODO uncomment
@ -168,6 +146,85 @@ public class ValidationController {
//xmlValidationResponse.setFairRules(fairRules);
}
private void getNumberOfRecords(String baseURL, int batchSize, Document oaipmhResponse, List<String> records, int recordsRetrieved) throws Exception {
String url;
HttpURLConnection conn;
while (recordsRetrieved < batchSize) {
Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0);
if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) {
break; // no more records to retrieve
}
url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent();
conn = (HttpURLConnection) new URL(url).openConnection();
oaipmhResponse = parseResponse(conn.getInputStream());
XPathFactory xfactory = XPathFactory.newInstance();
XPath xpath = xfactory.newXPath();
XPathExpression recordsExpression = xpath.compile("//record");
NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET);
for (int i = 0; i < recordNodes.getLength(); ++i) {
Node element = recordNodes.item(i);
StringWriter stringWriter = new StringWriter();
Transformer xform = TransformerFactory.newInstance().newTransformer();
xform.transform(new DOMSource(element), new StreamResult(stringWriter));
records.add(stringWriter.toString());
}
recordsRetrieved += records.size();
}
}
@RequestMapping(value = {"/realValidator"}, method = RequestMethod.GET)
public void validateWithApacheCamel(@RequestParam(name = "guidelines") String guidelinesProfileName,
@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
@RequestParam(name="set", required = false) String set, @RequestParam(name="metadataPrefix", defaultValue = "oai_dc") String metadataPrefix) {
AbstractOpenAireProfile profile = initializeOpenAireProfile(guidelinesProfileName);
AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName);
if (profile == null && fairProfile == null) {
log.error("Exception: No valid guidelines " + guidelinesProfileName + ".");
new Exception("Validation Job stopped unexpectedly. No valid guidelines " +
guidelinesProfileName + " were provided.");
}
ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords);
validationJob.guidelines = profile.name();
validationJobRepository.save(validationJob);
int record = 0;
double resultSum = 0;
try {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
CamelContext camelContext = new DefaultCamelContext();
RouteBuilder oaiPmhRouteBuilder =
new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix ,
profile, validationJob, numberOfRecords, validationIssueRepository, validationResultRepository);
camelContext.addRoutes(oaiPmhRouteBuilder);
camelContext.start();
//camelContext.stop();
validationJob.status = "COMPLETED";
}
catch (Exception e) {
log.error("Validation job stopped unexpectedly." + e.getMessage());
System.out.println("ERROR " + e.getMessage());
validationJob.status = "STOPPED";
} finally {
validationJob.endDate = new Date();
System.out.println("Final validation job "+ validationJob.hashCode());
validationJob.recordsTested = record;
validationJob.score = resultSum / record;
validationJobRepository.save(validationJob);
}
}
private void constructValidationJobResult(ValidationJob validationJob, ValidationRuleResult validationRuleResult,
Map.Entry entry, AbstractOpenAireProfile profile, Document document) {
@ -199,9 +256,8 @@ public class ValidationController {
}
private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
System.out.println("-------");
for (String error:engineResult.errors()) {
System.out.println("11111");
/*System.out.println("11111");*/
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
@ -209,11 +265,15 @@ public class ValidationController {
validationIssue.issueType = "ERROR";
validationIssue.issueText = error;
validationIssueRepository.save(validationIssue);
/*
System.out.println(validationIssue);
*/
}
for (String warning: engineResult.warnings()){
/*
System.out.println("22222");
*/
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
@ -221,7 +281,9 @@ public class ValidationController {
validationIssue.issueType = "WARNING";
validationIssue.issueText = warning;
validationIssueRepository.save(validationIssue);
/*
System.out.println(validationIssue);
*/
}
@ -360,7 +422,7 @@ public class ValidationController {
}
catch (Exception e) {
log.error("Validation job stopped unexpectedly.", e.getMessage());
log.error("Validation job stopped unexpectedly." + e.getMessage());
System.out.println("ERROR " + e.getMessage());
validationJob.status = "STOPPED";
@ -376,90 +438,7 @@ public class ValidationController {
//xmlValidationResponse.setFairRules(fairRules);
}
@RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.GET,
produces = MediaType.APPLICATION_JSON_VALUE)
public List<SummaryResult> getJobResults(@RequestParam(name = "jobId") int jobId){
System.out.println(validationResultRepository.getSummaryResult(jobId).toString());
System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult(jobId).get(0));
return validationResultRepository.getSummaryResult(jobId);
}
/*
@RequestMapping(value = {"getErrors"}, method = RequestMethod.GET,
produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getErrorsRecordUrls(jobId, ruleName);
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
for (Object[] result : resultList) {
String issueText = (String) result[0];
String recordUrl = (String) result[1];
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
if (recordUrls == null) {
recordUrls = new ArrayList<>();
recordUrlsByIssueText.put(issueText, recordUrls);
}
recordUrls.add(recordUrl);
}
System.out.println("issues " + recordUrlsByIssueText.keySet());
return recordUrlsByIssueText;
}*/
@RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET,
produces = MediaType.APPLICATION_JSON_VALUE)
public List<ValidationIssue> getRecordsByRule(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
return validationIssueRepository.getErrors(jobId, ruleName);
}
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET,
produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId){
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId);
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
for (Object[] result : resultList) {
String issueText = (String) result[0];
String recordUrl = (String) result[1];
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
if (recordUrls == null) {
recordUrls = new ArrayList<>();
recordUrlsByIssueText.put(issueText, recordUrls);
}
recordUrls.add(recordUrl);
}
System.out.println("issues " + recordUrlsByIssueText.keySet());
return recordUrlsByIssueText;
}
@RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET,
produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId);
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
for (Object[] result : resultList) {
String issueText = (String) result[0];
String recordUrl = (String) result[1];
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
if (recordUrls == null) {
recordUrls = new ArrayList<>();
recordUrlsByIssueText.put(issueText, recordUrls);
}
recordUrls.add(recordUrl);
}
System.out.println("issues " + recordUrlsByIssueText.keySet());
return recordUrlsByIssueText;
}
public List<String> extractRecordXmls(String xml) throws Exception {
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();

5
src/main/java/eu/dnetlib/validatorapi/controllers/ValidatorController.java
View File

@ -2,7 +2,6 @@ package eu.dnetlib.validatorapi.controllers;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.StandardResult;
import eu.dnetlib.validator2.validation.guideline.openaire.*;
import eu.dnetlib.validatorapi.entities.RuleInfo;
import eu.dnetlib.validatorapi.entities.XmlValidationResponse;
@ -15,7 +14,9 @@ import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.*;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.nio.file.Files;
import java.util.*;
import java.util.stream.Collectors;

2
src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java
View File

@ -1,7 +1,5 @@
package eu.dnetlib.validatorapi.entities;
import org.springframework.boot.autoconfigure.session.StoreType;
import javax.persistence.Entity;
import javax.persistence.Id;

186
src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java Normal file
View File

@ -0,0 +1,186 @@
package eu.dnetlib.validatorapi.processors;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.StringReader;
import java.util.Map;
public class XmlProcessor implements Processor {
private final AbstractOpenAireProfile profile;
private final ValidationJob validationJob;
private long processedRecords;
private final ValidationIssueRepository validationIssueRepository;
private final ValidationResultRepository validationResultRepository;
public XmlProcessor(final AbstractOpenAireProfile profile, final ValidationJob validationJob,
final ValidationIssueRepository validationIssueRepository,
final ValidationResultRepository validationResultRepository){
super();
this.profile = profile;
this.validationJob = validationJob;
this.validationIssueRepository = validationIssueRepository;
this.validationResultRepository = validationResultRepository;
}
@Override
public void process(Exchange exchange) throws Exception {
String recordXml = exchange.getIn().getBody(String.class);
final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
final DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml)));
String recordUrl = extractRecordUrl(doc, "identifier");
processedRecords++;
System.out.println("processed " + processedRecords);
exchange.setProperty("totalRecords", processedRecords);
if (profile != null) {
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
System.out.println("DC IDENTIFIER: " + recordUrl);
Map<String, Guideline.Result> results = validationResult.results();
ValidationRuleResult validationRuleResult;
for (Map.Entry entry : results.entrySet()) {
final Guideline.Result engineResult = (Guideline.Result) entry.getValue();
String ruleName = entry.getKey().toString();
validationRuleResult = new ValidationRuleResult();
constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
ruleName, profile, engineResult);
validationResultRepository.save(validationRuleResult);
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult);
// System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
}
}
}
private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
for (String error:engineResult.errors()) {
/*System.out.println("11111");*/
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
validationIssue.recordUrl = recordUrl;
validationIssue.issueType = "ERROR";
validationIssue.issueText = error;
validationIssueRepository.save(validationIssue);
/*
System.out.println(validationIssue);
*/
}
for (String warning: engineResult.warnings()){
/*
System.out.println("22222");
*/
ValidationIssue validationIssue = new ValidationIssue();
validationIssue.validationJobId = validationJobId;
validationIssue.ruleName = ruleName;
validationIssue.recordUrl = recordUrl;
validationIssue.issueType = "WARNING";
validationIssue.issueText = warning;
validationIssueRepository.save(validationIssue);
/*
System.out.println(validationIssue);
*/
}
}
/*public void process2(Exchange exchange) throws Exception {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
String recordXml = exchange.getIn().getBody(String.class);
// System.out.println("\n\n\n\n\n\n\n\n\n\n\n\n");
System.out.println(recordXml);
// System.out.println("\n\n\n\n\n\n\n\n\n\n\n\n");
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
if (profile != null) {
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
String recordUrl = extractRecordUrl(doc, "identifier");
System.out.println("DC IDENTIFIER: " + recordUrl);
Map<String, Guideline.Result> results = validationResult.results();
for (Map.Entry entry : results.entrySet()) {
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
// System.out.println("engine result > " + engineResult);
String ruleName = entry.getKey().toString();
// System.out.println("rulename: " + recordUrl);
ValidationRuleResult validationRuleResult = new ValidationRuleResult();
// System.out.println("validationjob.id " + validationJob.id);
// System.out.println("recordUrl " + recordUrl);
// System.out.println("rulename " + ruleName);
// System.out.println("profile " + profile);
// System.out.println("engineResult " + engineResult);
constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
ruleName, profile, engineResult);
// System.out.println("xaxaxa");
// System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
// resultSum += engineResult.score();
// elements = nodeListToList(recordNodes); // assign the Elements to the variable
}
}
processedRecords++;
System.out.println("processed " + processedRecords);
exchange.setProperty("totalRecords", processedRecords);
}*/
//TODO consider throwing exception - UTIL class?
private String extractRecordUrl(final Document document, final String xmlField) {
NodeList identifierNodes = document.getElementsByTagName(xmlField);
if (identifierNodes.getLength() > 0) {
Element identifierElement = (Element) identifierNodes.item(0);
return identifierElement.getTextContent();
}
return "-";
}
private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String recordUrl, String ruleName,
AbstractOpenAireProfile profile, Guideline.Result engineResult) {
validationRuleResult.validationJobId = validationJobId;
validationRuleResult.ruleName = ruleName;
validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight();
validationRuleResult.recordUrl = recordUrl;
validationRuleResult.score = engineResult.score();
validationRuleResult.status = engineResult.status().toString();
validationRuleResult.internalError = engineResult.internalError();
}
}

7
src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java
View File

@ -1,18 +1,12 @@
package eu.dnetlib.validatorapi.repositories;
import eu.dnetlib.validatorapi.entities.SummaryResult;
import eu.dnetlib.validatorapi.entities.ValidationIssue;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Repository
public interface ValidationIssueRepository extends JpaRepository<ValidationIssue, Long> {
@ -30,7 +24,6 @@ public interface ValidationIssueRepository extends JpaRepository<ValidationIssue
"GROUP BY vi.issueText, vi.recordUrl")
List<ValidationIssue> getErrors(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
"FROM ValidationIssue vi " +
"WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" +

130
src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java Normal file
View File

@ -0,0 +1,130 @@
package eu.dnetlib.validatorapi.routes;
import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
import eu.dnetlib.validatorapi.entities.ValidationJob;
import eu.dnetlib.validatorapi.processors.XmlProcessor;
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.apache.camel.builder.RouteBuilder;
public class OaiPmhRoute extends RouteBuilder {
private String oaiEndpoint = "oaipmh://http://repositorium.sdum.uminho.pt/oai/request";
private AbstractOpenAireProfile profile;
private long maxNumberOfRecords = 0;
private ValidationJob validationJob;
private final ValidationIssueRepository validationIssueRepository;
private final ValidationResultRepository validationResultRepository;
public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob,
long maxNumberOfRecords, final ValidationIssueRepository validationIssueRepository,
final ValidationResultRepository validationResultRepository) {
this.oaiEndpoint = oaiEndpoint;
this.validationJob = validationJob;
this.profile = profile;
this.maxNumberOfRecords = maxNumberOfRecords;
this.validationIssueRepository = validationIssueRepository;
this.validationResultRepository = validationResultRepository;
}
@Override
public void configure() throws Exception {
from(oaiEndpoint)
.setProperty("totalRecords", constant(0))
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 10")
.split(xpath("//*[local-name()='record']"))
// .log("\n\n\n----------------\n\n\n\n\n\n${body}\n\n\n----------------\n\n\n\n\n\n")
.process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository))
.end().to("direct:end");
/* from("oaipmh://http://repositorium.sdum.uminho.pt/oai/request?verb=ListRecords&metadataPrefix=oai_dc")
.setProperty("totalRecords", constant(0))
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 10")
.split(xpath("//*[local-name()='record']"))
//.log("\n\n\n----------------\n\n\n\n\n\n${body}\n\n\n----------------\n\n\n\n\n\n")
.process(new XmlProcessor(profile, validationJob))
.end().to("direct:end");*/
/*** kai auto from(oaiEndpoint) // trigger the route every minute
.split(xpath("//*[local-name()='record']"))
.setProperty("totalRecords", constant(0))
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 50")
.process(new XmlProcessor(profile, validationJob))
.end()
.to("direct:end");
**/
//from(oaiEndpoint).process(new XmlProcessor(profile,validationJob));
// from("direct:oaiRequest").process(new DummyXMLProcessor());
/*** AUTO PAIZEI !!!!!! ***/
/* from(oaiEndpoint).
split(xpath("//*[local-name()='record']")).process(new XmlProcessor(profile, validationJob));
*/
/*** KAI AUTO !!!!!! ***/
/*
from("timer://myTimer?fixedRate=true&period=60000") // trigger the route every minute
.setProperty("totalRecords", constant(0))
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 50")
.to("direct:oaiRequest")
.process(new RecordCountProcessor())
.end()
.to("direct:end");
from("direct:oaiRequest").
process(exchange -> {
int counter = exchange.getProperty("totalRecords", 0, Integer.class);
System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class));
counter++;
System.out.println("counter ++" + counter);
exchange.setProperty("processedRecords", counter);
System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class));
});
*/
/** KAI AUTO ***/
/*
from("timer://myTimer?fixedRate=true&period=60000") // trigger the route every minute
.setProperty("totalRecords", constant(0))
.loopDoWhile().simple("${exchangeProperty.totalRecords} < 50")
.to("direct:oaiRequest")
.process(new RecordCountProcessor())
.end()
.to("direct:end");
from("direct:oaiRequest").
process(exchange -> {
int counter = exchange.getProperty("totalRecords", 0, Integer.class);
System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class));
counter++;
System.out.println("counter ++" + counter);
exchange.setProperty("processedRecords", counter);
System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class));
});
*/
}
class RecordCountProcessor implements Processor {
@Override
public void process(Exchange exchange) throws Exception {
System.out.println("RecordCountProcessor");
int processedRecords = exchange.getProperty("processedRecords", 0, Integer.class);
exchange.setProperty("totalRecords", processedRecords);
}
}}

33
src/main/java/eu/dnetlib/validatorapi/utils/XmlProcessor.java
View File

@ -1,33 +0,0 @@
package eu.dnetlib.validatorapi.utils;
public class XmlProcessor { /* implements Processor {
private List<Element> elements;
@Override
public void process(Exchange exchange) throws Exception {
String xml = exchange.getIn().getBody(String.class);
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
Document doc = dBuilder.parse(new InputSource(new StringReader(xml)));
XPathFactory xfactory = XPathFactory.newInstance();
XPath xpath = xfactory.newXPath();
XPathExpression recordsExpression = xpath.compile("//record");
NodeList recordNodes = (NodeList) recordsExpression.evaluate(doc, XPathConstants.NODESET);
elements = nodeListToList(recordNodes); // assign the Elements to the variable
}
private List<Element> nodeListToList(NodeList nodeList) {
List<Element> list = new ArrayList<>();
for (int i = 0; i < nodeList.getLength(); i++) {
list.add((Element) nodeList.item(i));
}
return list;
}
public List<Element> getElements() {
return elements;
}*/
}

13
src/main/resources/log4j2.xml
View File

@ -6,6 +6,9 @@
</Property>
</Properties>
<Appenders>
<Console name="console" target="SYSTEM_OUT" follow="true">
<PatternLayout pattern="${LOG_PATTERN}"/>
</Console>
<!-- Rolling File Appender -->
<RollingFile name="R" fileName="/var/log/dnet/uoa-validator-api/uoa-validator-api.log"
filePattern="/var/log/dnet/uoa-validator-api/uoa-validator-api-%d{yyyy-MM-dd}-%i.log">
@ -29,11 +32,17 @@
</RollingFile>
</Appenders>
<Loggers>
<Logger name="org.springframework" level="debug" additivity="false">
<Logger name="org.springframework" level="info" additivity="false">
<AppenderRef ref="S"/>
<AppenderRef ref="console"/>
</Logger>
<Root level="debug">
<Logger name="org.springframework" level="info" additivity="false">
<AppenderRef ref="S"/>
<AppenderRef ref="console"/>
</Logger>
<Root level="info">
<AppenderRef ref="R"/>
<AppenderRef ref="console"/>
</Root>
</Loggers>
</Configuration>

3
src/main/resources/records.xml
View File

@ -10598,7 +10598,8 @@
<setSpec>publications:year2017</setSpec>
</header>
<metadata>
<oai_dc:dc xmlns="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:ns2="http://purl.org/dc/terms/">
<oai_dc:dc xmlns="http://purl.org/dc/elements/1.1/"
xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
<title xml:lang="eng">Report on challenges for SCIs:Smart Resilience Indicators for Smart Critical Infrastructures Deliverable D2.2</title>
<creator>Walther, G.</creator>
<creator>Jovanovic, M.</creator>