From fe7a8b16315690d26a167a4a7bc8be545d33f22e Mon Sep 17 00:00:00 2001 From: Katerina Date: Mon, 29 May 2023 14:40:25 +0300 Subject: [PATCH] progress with apache camel dynamic routes --- pom.xml | 14 + .../controllers/ReportController.java | 74 ++++++ .../controllers/ValidationController.java | 239 ++++++++---------- .../controllers/ValidatorController.java | 5 +- .../validatorapi/entities/SummaryResult.java | 2 - .../validatorapi/processors/XmlProcessor.java | 186 ++++++++++++++ .../ValidationIssueRepository.java | 7 - .../validatorapi/routes/OaiPmhRoute.java | 130 ++++++++++ .../validatorapi/utils/XmlProcessor.java | 33 --- src/main/resources/log4j2.xml | 13 +- src/main/resources/records.xml | 3 +- 11 files changed, 529 insertions(+), 177 deletions(-) create mode 100644 src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java create mode 100644 src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java create mode 100644 src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java delete mode 100644 src/main/java/eu/dnetlib/validatorapi/utils/XmlProcessor.java diff --git a/pom.xml b/pom.xml index 92d826f..fb7499e 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ UTF-8 ${maven.build.timestamp} E MMM dd HH:mm:ss z yyyy + 11 @@ -40,6 +41,18 @@ org.postgresql postgresql + + + + org.apache.camel + camel-core + 3.20.3 + + + org.apache.camel + camel-oaipmh + 3.20.3 + @@ -47,6 +60,7 @@ org.springframework.boot spring-boot-maven-plugin + 1.5.8.RELEASE eu.dnetlib.validatorapi.ValidatorApiApplication true diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java new file mode 100644 index 0000000..b63ff27 --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java @@ -0,0 +1,74 @@ +package eu.dnetlib.validatorapi.controllers; + +import eu.dnetlib.validatorapi.entities.SummaryResult; +import eu.dnetlib.validatorapi.entities.ValidationIssue; +import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; +import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; +import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RestController +@CrossOrigin(origins = "*") +@RequestMapping("/reports") +public class ReportController { + private final ValidationJobRepository validationJobRepository; + private final ValidationResultRepository validationResultRepository; + private final ValidationIssueRepository validationIssueRepository; + + public ReportController(ValidationJobRepository validationJobRepository, + ValidationResultRepository validationResultRepository, + ValidationIssueRepository validationIssueRepository) { + this.validationJobRepository = validationJobRepository; + this.validationResultRepository = validationResultRepository; + this.validationIssueRepository = validationIssueRepository; + } + + + @RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public List getJobResults(@RequestParam(name = "jobId") int jobId){ + return validationResultRepository.getSummaryResult(jobId); + } + + @RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public List getRecordsByRule(@RequestParam(name = "jobId") int jobId, + @RequestParam(name = "ruleName") String ruleName){ + return validationIssueRepository.getErrors(jobId, ruleName); + } + + @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public Map> getErrors(@RequestParam(name = "jobId") int jobId){ + List resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId); + return extractRecordsGroupedByRule(resultList); + } + + @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public Map> getWarningsReport(@RequestParam(name = "jobId") int jobId, + @RequestParam(name = "ruleName") String ruleName){ + List resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId); + return extractRecordsGroupedByRule(resultList); + } + + private Map> extractRecordsGroupedByRule(List resultList) { + Map> recordUrlsByIssueText = new HashMap<>(); + + for (Object[] result : resultList) { + String issueText = (String) result[0]; + String recordUrl = (String) result[1]; + + List recordUrls = recordUrlsByIssueText.get(issueText); + if (recordUrls == null) { + recordUrls = new ArrayList<>(); + recordUrlsByIssueText.put(issueText, recordUrls); + } + recordUrls.add(recordUrl); + } + return recordUrlsByIssueText; + } + +} diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index 456be40..dfacc67 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -3,14 +3,20 @@ package eu.dnetlib.validatorapi.controllers; import eu.dnetlib.validator2.validation.XMLApplicationProfile; import eu.dnetlib.validator2.validation.guideline.Guideline; import eu.dnetlib.validator2.validation.guideline.openaire.*; -import eu.dnetlib.validatorapi.entities.*; +import eu.dnetlib.validatorapi.entities.RuleInfo; +import eu.dnetlib.validatorapi.entities.ValidationIssue; +import eu.dnetlib.validatorapi.entities.ValidationJob; +import eu.dnetlib.validatorapi.entities.ValidationRuleResult; import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; +import eu.dnetlib.validatorapi.routes.OaiPmhRoute; +import org.apache.camel.CamelContext; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.impl.DefaultCamelContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -34,7 +40,10 @@ import java.io.StringReader; import java.io.StringWriter; import java.net.HttpURLConnection; import java.net.URL; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; @RestController @CrossOrigin(origins = "*") @@ -46,13 +55,15 @@ public class ValidationController { private final ValidationIssueRepository validationIssueRepository; @Autowired - public ValidationController(ValidationJobRepository validationJobRepository, ValidationResultRepository validationResultRepository, ValidationIssueRepository validationIssueRepository) { + public ValidationController(ValidationJobRepository validationJobRepository, + ValidationResultRepository validationResultRepository, + ValidationIssueRepository validationIssueRepository) { this.validationJobRepository = validationJobRepository; this.validationResultRepository = validationResultRepository; this.validationIssueRepository = validationIssueRepository; } - @RequestMapping(value = {"/realValidator"}, method = RequestMethod.POST) + @RequestMapping(value = {"/realValidator-old"}, method = RequestMethod.GET) public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName, @RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now @@ -74,7 +85,6 @@ public class ValidationController { double resultSum = 0; try { - System.out.println("making request...."); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); @@ -82,46 +92,19 @@ public class ValidationController { if (set != null) { url += "&set=" + set; } - System.out.println("URL " + url.toString()); HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); Document oaipmhResponse = parseResponse(conn.getInputStream()); - List records = new ArrayList(); + //TODO: follow other approach if records are more than 100!!! int recordsRetrieved = 0; - // Step 2: Retrieve additional records using resumptionToken - while (recordsRetrieved < batchSize) { - System.out.println("True"); - Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0); - if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) { - System.out.println("breaking"); - break; // no more records to retrieve - } - url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent(); - conn = (HttpURLConnection) new URL(url).openConnection(); - oaipmhResponse = parseResponse(conn.getInputStream()); - - XPathFactory xfactory = XPathFactory.newInstance(); - XPath xpath = xfactory.newXPath(); - XPathExpression recordsExpression = xpath.compile("//record"); - NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET); - - for (int i = 0; i < recordNodes.getLength(); ++i) { - Node element = recordNodes.item(i); - StringWriter stringWriter = new StringWriter(); - Transformer xform = TransformerFactory.newInstance().newTransformer(); - xform.transform(new DOMSource(element), new StreamResult(stringWriter)); - records.add(stringWriter.toString()); - } - - recordsRetrieved += records.size(); - } + List records = new ArrayList(); + // Retrieve additional records using resumptionToken + getNumberOfRecords(baseURL, batchSize, oaipmhResponse, records, recordsRetrieved); for (String recordXml : records) { - System.out.println("\n\n\n\n\n\n\n\n HELLLOOOOOOOOOO \n\n\n\n\n"); Document doc = db.parse(new InputSource(new StringReader(recordXml))); - System.out.println("\n\n\n\n RECORD : " + recordXml + "\n\n\n\n\n"); if (profile != null) { XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that? @@ -130,13 +113,9 @@ public class ValidationController { for (Map.Entry entry : results.entrySet()) { Guideline.Result engineResult = (Guideline.Result) entry.getValue(); String recordUrl = extractRecordUrl(doc, "identifier"); - System.out.println("DC IDENTIFIER: " + recordUrl); String ruleName = entry.getKey().toString(); ValidationRuleResult validationRuleResult = constructValidationRuleResult(validationJob.id, recordUrl, ruleName, profile, engineResult); - - System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n"); - resultSum += engineResult.score(); validationResultRepository.save(validationRuleResult); @@ -144,20 +123,19 @@ public class ValidationController { } record++; - System.out.println(record); } } validationJob.status = "COMPLETED"; } catch (Exception e) { - log.error("Validation job stopped unexpectedly.", e.getMessage()); + log.error("Validation job stopped unexpectedly." + e.getMessage()); System.out.println("ERROR " + e.getMessage()); validationJob.status = "STOPPED"; } finally { validationJob.endDate = new Date(); - System.out.println("Final validation job "+ validationJob.hashCode()); + //System.out.println("Final validation job "+ validationJob.hashCode()); validationJob.recordsTested = record; validationJob.score = resultSum / record; //TODO uncomment @@ -168,6 +146,85 @@ public class ValidationController { //xmlValidationResponse.setFairRules(fairRules); } + private void getNumberOfRecords(String baseURL, int batchSize, Document oaipmhResponse, List records, int recordsRetrieved) throws Exception { + String url; + HttpURLConnection conn; + while (recordsRetrieved < batchSize) { + Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0); + if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) { + break; // no more records to retrieve + } + url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent(); + conn = (HttpURLConnection) new URL(url).openConnection(); + oaipmhResponse = parseResponse(conn.getInputStream()); + + XPathFactory xfactory = XPathFactory.newInstance(); + XPath xpath = xfactory.newXPath(); + XPathExpression recordsExpression = xpath.compile("//record"); + NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET); + + for (int i = 0; i < recordNodes.getLength(); ++i) { + Node element = recordNodes.item(i); + StringWriter stringWriter = new StringWriter(); + Transformer xform = TransformerFactory.newInstance().newTransformer(); + xform.transform(new DOMSource(element), new StreamResult(stringWriter)); + records.add(stringWriter.toString()); + } + + recordsRetrieved += records.size(); + } + } + + @RequestMapping(value = {"/realValidator"}, method = RequestMethod.GET) + public void validateWithApacheCamel(@RequestParam(name = "guidelines") String guidelinesProfileName, + @RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now + @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now + @RequestParam(name="set", required = false) String set, @RequestParam(name="metadataPrefix", defaultValue = "oai_dc") String metadataPrefix) { + + AbstractOpenAireProfile profile = initializeOpenAireProfile(guidelinesProfileName); + AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName); + + if (profile == null && fairProfile == null) { + log.error("Exception: No valid guidelines " + guidelinesProfileName + "."); + new Exception("Validation Job stopped unexpectedly. No valid guidelines " + + guidelinesProfileName + " were provided."); + } + + ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords); + validationJob.guidelines = profile.name(); + validationJobRepository.save(validationJob); + + int record = 0; + double resultSum = 0; + + try { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + CamelContext camelContext = new DefaultCamelContext(); + RouteBuilder oaiPmhRouteBuilder = + new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix , + profile, validationJob, numberOfRecords, validationIssueRepository, validationResultRepository); + camelContext.addRoutes(oaiPmhRouteBuilder); + camelContext.start(); + //camelContext.stop(); + validationJob.status = "COMPLETED"; + + } + catch (Exception e) { + log.error("Validation job stopped unexpectedly." + e.getMessage()); + System.out.println("ERROR " + e.getMessage()); + validationJob.status = "STOPPED"; + + } finally { + validationJob.endDate = new Date(); + System.out.println("Final validation job "+ validationJob.hashCode()); + validationJob.recordsTested = record; + validationJob.score = resultSum / record; + validationJobRepository.save(validationJob); + } + + } + private void constructValidationJobResult(ValidationJob validationJob, ValidationRuleResult validationRuleResult, Map.Entry entry, AbstractOpenAireProfile profile, Document document) { @@ -199,9 +256,8 @@ public class ValidationController { } private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) { - System.out.println("-------"); for (String error:engineResult.errors()) { - System.out.println("11111"); + /*System.out.println("11111");*/ ValidationIssue validationIssue = new ValidationIssue(); validationIssue.validationJobId = validationJobId; validationIssue.ruleName = ruleName; @@ -209,11 +265,15 @@ public class ValidationController { validationIssue.issueType = "ERROR"; validationIssue.issueText = error; validationIssueRepository.save(validationIssue); +/* System.out.println(validationIssue); +*/ } for (String warning: engineResult.warnings()){ +/* System.out.println("22222"); +*/ ValidationIssue validationIssue = new ValidationIssue(); validationIssue.validationJobId = validationJobId; validationIssue.ruleName = ruleName; @@ -221,7 +281,9 @@ public class ValidationController { validationIssue.issueType = "WARNING"; validationIssue.issueText = warning; validationIssueRepository.save(validationIssue); +/* System.out.println(validationIssue); +*/ } @@ -360,7 +422,7 @@ public class ValidationController { } catch (Exception e) { - log.error("Validation job stopped unexpectedly.", e.getMessage()); + log.error("Validation job stopped unexpectedly." + e.getMessage()); System.out.println("ERROR " + e.getMessage()); validationJob.status = "STOPPED"; @@ -376,90 +438,7 @@ public class ValidationController { //xmlValidationResponse.setFairRules(fairRules); } - @RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.GET, - produces = MediaType.APPLICATION_JSON_VALUE) - public List getJobResults(@RequestParam(name = "jobId") int jobId){ - System.out.println(validationResultRepository.getSummaryResult(jobId).toString()); - System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult(jobId).get(0)); - return validationResultRepository.getSummaryResult(jobId); - } -/* - @RequestMapping(value = {"getErrors"}, method = RequestMethod.GET, - produces = MediaType.APPLICATION_JSON_VALUE) - public Map> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ - List resultList = validationIssueRepository.getErrorsRecordUrls(jobId, ruleName); - - Map> recordUrlsByIssueText = new HashMap<>(); - - for (Object[] result : resultList) { - String issueText = (String) result[0]; - String recordUrl = (String) result[1]; - - List recordUrls = recordUrlsByIssueText.get(issueText); - if (recordUrls == null) { - recordUrls = new ArrayList<>(); - recordUrlsByIssueText.put(issueText, recordUrls); - } - recordUrls.add(recordUrl); - } - - System.out.println("issues " + recordUrlsByIssueText.keySet()); - - return recordUrlsByIssueText; - }*/ - - @RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, - produces = MediaType.APPLICATION_JSON_VALUE) - public List getRecordsByRule(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ - return validationIssueRepository.getErrors(jobId, ruleName); - } - - @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, - produces = MediaType.APPLICATION_JSON_VALUE) - public Map> getErrors(@RequestParam(name = "jobId") int jobId){ - List resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId); - Map> recordUrlsByIssueText = new HashMap<>(); - - for (Object[] result : resultList) { - String issueText = (String) result[0]; - String recordUrl = (String) result[1]; - - List recordUrls = recordUrlsByIssueText.get(issueText); - if (recordUrls == null) { - recordUrls = new ArrayList<>(); - recordUrlsByIssueText.put(issueText, recordUrls); - } - recordUrls.add(recordUrl); - } - - System.out.println("issues " + recordUrlsByIssueText.keySet()); - - return recordUrlsByIssueText; - } - - @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, - produces = MediaType.APPLICATION_JSON_VALUE) - public Map> getWarningsReport(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ - List resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId); - Map> recordUrlsByIssueText = new HashMap<>(); - - for (Object[] result : resultList) { - String issueText = (String) result[0]; - String recordUrl = (String) result[1]; - - List recordUrls = recordUrlsByIssueText.get(issueText); - if (recordUrls == null) { - recordUrls = new ArrayList<>(); - recordUrlsByIssueText.put(issueText, recordUrls); - } - recordUrls.add(recordUrl); - } - - System.out.println("issues " + recordUrlsByIssueText.keySet()); - - return recordUrlsByIssueText; - } public List extractRecordXmls(String xml) throws Exception { DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidatorController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidatorController.java index db9809f..f6b9c7a 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidatorController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidatorController.java @@ -2,7 +2,6 @@ package eu.dnetlib.validatorapi.controllers; import eu.dnetlib.validator2.validation.XMLApplicationProfile; import eu.dnetlib.validator2.validation.guideline.Guideline; -import eu.dnetlib.validator2.validation.guideline.StandardResult; import eu.dnetlib.validator2.validation.guideline.openaire.*; import eu.dnetlib.validatorapi.entities.RuleInfo; import eu.dnetlib.validatorapi.entities.XmlValidationResponse; @@ -15,7 +14,9 @@ import org.xml.sax.InputSource; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; import java.nio.file.Files; import java.util.*; import java.util.stream.Collectors; diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java b/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java index 9c4b781..7f75821 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java @@ -1,7 +1,5 @@ package eu.dnetlib.validatorapi.entities; -import org.springframework.boot.autoconfigure.session.StoreType; - import javax.persistence.Entity; import javax.persistence.Id; diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java new file mode 100644 index 0000000..d0babd3 --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java @@ -0,0 +1,186 @@ +package eu.dnetlib.validatorapi.processors; + +import eu.dnetlib.validator2.validation.XMLApplicationProfile; +import eu.dnetlib.validator2.validation.guideline.Guideline; +import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile; +import eu.dnetlib.validatorapi.entities.ValidationIssue; +import eu.dnetlib.validatorapi.entities.ValidationJob; +import eu.dnetlib.validatorapi.entities.ValidationRuleResult; +import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; +import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; +import org.apache.camel.Exchange; +import org.apache.camel.Processor; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.StringReader; +import java.util.Map; + +public class XmlProcessor implements Processor { + private final AbstractOpenAireProfile profile; + private final ValidationJob validationJob; + private long processedRecords; + + private final ValidationIssueRepository validationIssueRepository; + private final ValidationResultRepository validationResultRepository; + + public XmlProcessor(final AbstractOpenAireProfile profile, final ValidationJob validationJob, + final ValidationIssueRepository validationIssueRepository, + final ValidationResultRepository validationResultRepository){ + super(); + this.profile = profile; + this.validationJob = validationJob; + this.validationIssueRepository = validationIssueRepository; + this.validationResultRepository = validationResultRepository; + } + + + @Override + public void process(Exchange exchange) throws Exception { + String recordXml = exchange.getIn().getBody(String.class); + + final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + final DocumentBuilder documentBuilder = dbf.newDocumentBuilder(); + Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml))); + String recordUrl = extractRecordUrl(doc, "identifier"); + + processedRecords++; + System.out.println("processed " + processedRecords); + exchange.setProperty("totalRecords", processedRecords); + + if (profile != null) { + XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that? + System.out.println("DC IDENTIFIER: " + recordUrl); + + Map results = validationResult.results(); + + ValidationRuleResult validationRuleResult; + for (Map.Entry entry : results.entrySet()) { + final Guideline.Result engineResult = (Guideline.Result) entry.getValue(); + String ruleName = entry.getKey().toString(); + validationRuleResult = new ValidationRuleResult(); + constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl, + ruleName, profile, engineResult); + validationResultRepository.save(validationRuleResult); + saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult); + + // System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n"); + } + + } + } + + private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) { + for (String error:engineResult.errors()) { + /*System.out.println("11111");*/ + ValidationIssue validationIssue = new ValidationIssue(); + validationIssue.validationJobId = validationJobId; + validationIssue.ruleName = ruleName; + validationIssue.recordUrl = recordUrl; + validationIssue.issueType = "ERROR"; + validationIssue.issueText = error; + validationIssueRepository.save(validationIssue); +/* + System.out.println(validationIssue); +*/ + } + + for (String warning: engineResult.warnings()){ +/* + System.out.println("22222"); +*/ + ValidationIssue validationIssue = new ValidationIssue(); + validationIssue.validationJobId = validationJobId; + validationIssue.ruleName = ruleName; + validationIssue.recordUrl = recordUrl; + validationIssue.issueType = "WARNING"; + validationIssue.issueText = warning; + validationIssueRepository.save(validationIssue); +/* + System.out.println(validationIssue); +*/ + + } + + } + + /*public void process2(Exchange exchange) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + String recordXml = exchange.getIn().getBody(String.class); + + // System.out.println("\n\n\n\n\n\n\n\n\n\n\n\n"); + System.out.println(recordXml); + // System.out.println("\n\n\n\n\n\n\n\n\n\n\n\n"); + + Document doc = db.parse(new InputSource(new StringReader(recordXml))); + + + if (profile != null) { + XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that? + String recordUrl = extractRecordUrl(doc, "identifier"); + System.out.println("DC IDENTIFIER: " + recordUrl); + + Map results = validationResult.results(); + for (Map.Entry entry : results.entrySet()) { + Guideline.Result engineResult = (Guideline.Result) entry.getValue(); + // System.out.println("engine result > " + engineResult); + + String ruleName = entry.getKey().toString(); + // System.out.println("rulename: " + recordUrl); + + ValidationRuleResult validationRuleResult = new ValidationRuleResult(); + // System.out.println("validationjob.id " + validationJob.id); + // System.out.println("recordUrl " + recordUrl); + // System.out.println("rulename " + ruleName); + // System.out.println("profile " + profile); + // System.out.println("engineResult " + engineResult); + + + + constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl, + ruleName, profile, engineResult); + + // System.out.println("xaxaxa"); + // System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n"); + // resultSum += engineResult.score(); + + // elements = nodeListToList(recordNodes); // assign the Elements to the variable + + + + } + } + + processedRecords++; + System.out.println("processed " + processedRecords); + exchange.setProperty("totalRecords", processedRecords); + }*/ + + //TODO consider throwing exception - UTIL class? + private String extractRecordUrl(final Document document, final String xmlField) { + NodeList identifierNodes = document.getElementsByTagName(xmlField); + if (identifierNodes.getLength() > 0) { + Element identifierElement = (Element) identifierNodes.item(0); + return identifierElement.getTextContent(); + } + return "-"; + } + + + private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String recordUrl, String ruleName, + AbstractOpenAireProfile profile, Guideline.Result engineResult) { + + validationRuleResult.validationJobId = validationJobId; + validationRuleResult.ruleName = ruleName; + validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight(); + validationRuleResult.recordUrl = recordUrl; + validationRuleResult.score = engineResult.score(); + validationRuleResult.status = engineResult.status().toString(); + validationRuleResult.internalError = engineResult.internalError(); + } +} diff --git a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java index 626016b..6198041 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java @@ -1,18 +1,12 @@ package eu.dnetlib.validatorapi.repositories; -import eu.dnetlib.validatorapi.entities.SummaryResult; import eu.dnetlib.validatorapi.entities.ValidationIssue; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; @Repository public interface ValidationIssueRepository extends JpaRepository { @@ -30,7 +24,6 @@ public interface ValidationIssueRepository extends JpaRepository getErrors(@Param("id") int validationJobId, @Param("ruleName") String ruleName); - @Query(value = "SELECT vi.issueText, vi.recordUrl " + "FROM ValidationIssue vi " + "WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" + diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java new file mode 100644 index 0000000..a1e14ec --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java @@ -0,0 +1,130 @@ +package eu.dnetlib.validatorapi.routes; + +import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile; +import eu.dnetlib.validatorapi.entities.ValidationJob; +import eu.dnetlib.validatorapi.processors.XmlProcessor; +import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; +import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; +import org.apache.camel.Exchange; +import org.apache.camel.Processor; +import org.apache.camel.builder.RouteBuilder; + +public class OaiPmhRoute extends RouteBuilder { + + private String oaiEndpoint = "oaipmh://http://repositorium.sdum.uminho.pt/oai/request"; + + private AbstractOpenAireProfile profile; + private long maxNumberOfRecords = 0; + private ValidationJob validationJob; + private final ValidationIssueRepository validationIssueRepository; + private final ValidationResultRepository validationResultRepository; + + public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob, + long maxNumberOfRecords, final ValidationIssueRepository validationIssueRepository, + final ValidationResultRepository validationResultRepository) { + this.oaiEndpoint = oaiEndpoint; + this.validationJob = validationJob; + this.profile = profile; + this.maxNumberOfRecords = maxNumberOfRecords; + this.validationIssueRepository = validationIssueRepository; + this.validationResultRepository = validationResultRepository; + } + + @Override + public void configure() throws Exception { + + + from(oaiEndpoint) + .setProperty("totalRecords", constant(0)) + .loopDoWhile().simple("${exchangeProperty.totalRecords} < 10") + .split(xpath("//*[local-name()='record']")) + // .log("\n\n\n----------------\n\n\n\n\n\n${body}\n\n\n----------------\n\n\n\n\n\n") + .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository)) + .end().to("direct:end"); + + /* from("oaipmh://http://repositorium.sdum.uminho.pt/oai/request?verb=ListRecords&metadataPrefix=oai_dc") + .setProperty("totalRecords", constant(0)) + .loopDoWhile().simple("${exchangeProperty.totalRecords} < 10") + .split(xpath("//*[local-name()='record']")) + //.log("\n\n\n----------------\n\n\n\n\n\n${body}\n\n\n----------------\n\n\n\n\n\n") + .process(new XmlProcessor(profile, validationJob)) + .end().to("direct:end");*/ + + + +/*** kai auto from(oaiEndpoint) // trigger the route every minute + .split(xpath("//*[local-name()='record']")) + .setProperty("totalRecords", constant(0)) + .loopDoWhile().simple("${exchangeProperty.totalRecords} < 50") + .process(new XmlProcessor(profile, validationJob)) + .end() + .to("direct:end"); +**/ + //from(oaiEndpoint).process(new XmlProcessor(profile,validationJob)); + + // from("direct:oaiRequest").process(new DummyXMLProcessor()); + + + + + /*** AUTO PAIZEI !!!!!! ***/ +/* from(oaiEndpoint). + split(xpath("//*[local-name()='record']")).process(new XmlProcessor(profile, validationJob)); +*/ + + /*** KAI AUTO !!!!!! ***/ + /* + from("timer://myTimer?fixedRate=true&period=60000") // trigger the route every minute + .setProperty("totalRecords", constant(0)) + .loopDoWhile().simple("${exchangeProperty.totalRecords} < 50") + .to("direct:oaiRequest") + .process(new RecordCountProcessor()) + .end() + .to("direct:end"); + + from("direct:oaiRequest"). + process(exchange -> { + int counter = exchange.getProperty("totalRecords", 0, Integer.class); + System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class)); + counter++; + System.out.println("counter ++" + counter); + exchange.setProperty("processedRecords", counter); + System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class)); + + }); + */ + + /** KAI AUTO ***/ + /* + from("timer://myTimer?fixedRate=true&period=60000") // trigger the route every minute + .setProperty("totalRecords", constant(0)) + .loopDoWhile().simple("${exchangeProperty.totalRecords} < 50") + .to("direct:oaiRequest") + .process(new RecordCountProcessor()) + .end() + .to("direct:end"); + + from("direct:oaiRequest"). + process(exchange -> { + int counter = exchange.getProperty("totalRecords", 0, Integer.class); + System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class)); + counter++; + System.out.println("counter ++" + counter); + exchange.setProperty("processedRecords", counter); + System.out.println("Processing iteration: " + exchange.getProperty("totalRecords", 0, Integer.class)); + + }); + */ + } + + + class RecordCountProcessor implements Processor { + @Override + public void process(Exchange exchange) throws Exception { + System.out.println("RecordCountProcessor"); + + int processedRecords = exchange.getProperty("processedRecords", 0, Integer.class); + exchange.setProperty("totalRecords", processedRecords); + + } + }} \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/validatorapi/utils/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/utils/XmlProcessor.java deleted file mode 100644 index 881558b..0000000 --- a/src/main/java/eu/dnetlib/validatorapi/utils/XmlProcessor.java +++ /dev/null @@ -1,33 +0,0 @@ -package eu.dnetlib.validatorapi.utils; - -public class XmlProcessor { /* implements Processor { - private List elements; - - @Override - public void process(Exchange exchange) throws Exception { - String xml = exchange.getIn().getBody(String.class); - - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - Document doc = dBuilder.parse(new InputSource(new StringReader(xml))); - - XPathFactory xfactory = XPathFactory.newInstance(); - XPath xpath = xfactory.newXPath(); - XPathExpression recordsExpression = xpath.compile("//record"); - NodeList recordNodes = (NodeList) recordsExpression.evaluate(doc, XPathConstants.NODESET); - - elements = nodeListToList(recordNodes); // assign the Elements to the variable - } - - private List nodeListToList(NodeList nodeList) { - List list = new ArrayList<>(); - for (int i = 0; i < nodeList.getLength(); i++) { - list.add((Element) nodeList.item(i)); - } - return list; - } - - public List getElements() { - return elements; - }*/ -} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 15ce9dd..f1abce9 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -6,6 +6,9 @@ + + + @@ -29,11 +32,17 @@ - + + - + + + + + + \ No newline at end of file diff --git a/src/main/resources/records.xml b/src/main/resources/records.xml index bb6dfc0..d6b53db 100644 --- a/src/main/resources/records.xml +++ b/src/main/resources/records.xml @@ -10598,7 +10598,8 @@ publications:year2017 - + Report on challenges for SCIs:Smart Resilience Indicators for Smart Critical Infrastructures Deliverable D2.2 Walther, G. Jovanovic, M.