From ec549036d93c87a97189f74023df81457960f215 Mon Sep 17 00:00:00 2001 From: Katerina Date: Fri, 14 Jul 2023 14:21:21 +0300 Subject: [PATCH] Steps to remove save methods from Processors. Added apache camel-jpa dependency --- pom.xml | 9 ++++ .../controllers/ValidationController.java | 37 +++++++------ .../entities/ValidationIssue.java | 10 ++++ .../processors/DataBaseProcessor.java | 8 +-- .../validatorapi/processors/XmlProcessor.java | 53 +++++++++---------- .../validatorapi/routes/OaiPmhRoute.java | 22 ++++++-- .../validatorapi/routes/OaiSetListRoute.java | 19 ++----- 7 files changed, 84 insertions(+), 74 deletions(-) diff --git a/pom.xml b/pom.xml index eb0e8f1..f175054 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,15 @@ camel-http ${apache.camel.version} + + org.apache.camel + camel-jpa + ${apache.camel.version} + + + + + diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index d440224..ef88b80 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -4,14 +4,12 @@ import eu.dnetlib.validator2.validation.XMLApplicationProfile; import eu.dnetlib.validator2.validation.guideline.Guideline; import eu.dnetlib.validator2.validation.guideline.openaire.*; import eu.dnetlib.validatorapi.entities.RuleInfo; -import eu.dnetlib.validatorapi.entities.ValidationIssue; import eu.dnetlib.validatorapi.entities.ValidationJob; import eu.dnetlib.validatorapi.entities.ValidationRuleResult; import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import eu.dnetlib.validatorapi.routes.OaiPmhRoute; -import eu.dnetlib.validatorapi.routes.OaiSetListRoute; import org.apache.camel.CamelContext; import org.apache.camel.ProducerTemplate; import org.apache.camel.builder.RouteBuilder; @@ -124,7 +122,7 @@ public class ValidationController { resultSum += engineResult.score(); validationResultRepository.save(validationRuleResult); - saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult); + //saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult); } record++; @@ -212,10 +210,15 @@ public class ValidationController { DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); + RouteBuilder oaiPmhRouteBuilder = new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix , - profile, validationJob, numberOfRecords, validationJobRepository, - validationIssueRepository, validationResultRepository); + profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository); + + /* + RouteBuilder oaiPmhRouteBuilder = + new OAI_PMH_RouteBuilder("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix , + profile, validationJob, numberOfRecords, validationJobRepository, validationIssueRepository, validationResultRepository);*/ camelContext.addRoutes(oaiPmhRouteBuilder); } @@ -234,13 +237,9 @@ public class ValidationController { public String getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now ) throws Exception { - RouteBuilder listSetsBuilder = new OaiSetListRoute(); ProducerTemplate producerTemplate = camelContext.createProducerTemplate(); String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class); -/* - ConsumerTemplate consumerTemplate = camelContext.createConsumerTemplate(); - String result = consumerTemplate.receive("seda:result").getIn().getBody(String.class); - return result;*/ + return response; /* @@ -298,9 +297,9 @@ public class ValidationController { return validationRuleResult; } - private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) { +/* private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) { for (String error:engineResult.errors()) { - /*System.out.println("11111");*/ + *//*System.out.println("11111");*//* ValidationIssue validationIssue = new ValidationIssue(); validationIssue.validationJobId = validationJobId; validationIssue.ruleName = ruleName; @@ -308,15 +307,15 @@ public class ValidationController { validationIssue.issueType = "ERROR"; validationIssue.issueText = error; validationIssueRepository.save(validationIssue); -/* +*//* System.out.println(validationIssue); -*/ +*//* } for (String warning: engineResult.warnings()){ -/* +*//* System.out.println("22222"); -*/ +*//* ValidationIssue validationIssue = new ValidationIssue(); validationIssue.validationJobId = validationJobId; validationIssue.ruleName = ruleName; @@ -324,13 +323,13 @@ public class ValidationController { validationIssue.issueType = "WARNING"; validationIssue.issueText = warning; validationIssueRepository.save(validationIssue); -/* +*//* System.out.println(validationIssue); -*/ +*//* } - } + }*/ private ValidationRuleResult constructValidationRuleResult(int validationJobId, String recordUrl, String ruleName, AbstractOpenAireProfile profile, Guideline.Result engineResult) { diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java index c9fba9a..09f7344 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java @@ -26,6 +26,16 @@ public class ValidationIssue implements Serializable { @Column(name = "issue_type") public String issueType; + public ValidationIssue(){}; + + public ValidationIssue(int validationJobId, String ruleName, String recordUrl, String issueText, String issueType) { + this.validationJobId = validationJobId; + this.ruleName = ruleName; + this.recordUrl = recordUrl; + this.issueText = issueText; + this.issueType = issueType; + } + public int getValidationJobId() { return validationJobId; } diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java index 0aca156..93de892 100644 --- a/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java +++ b/src/main/java/eu/dnetlib/validatorapi/processors/DataBaseProcessor.java @@ -1,7 +1,6 @@ package eu.dnetlib.validatorapi.processors; import eu.dnetlib.validatorapi.entities.ValidationJob; -import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import org.apache.camel.Exchange; import org.apache.camel.Processor; @@ -10,13 +9,11 @@ import java.util.Date; public class DataBaseProcessor implements Processor { private final ValidationJob validationJob; - private final ValidationJobRepository validationJobRepository; private final ValidationResultRepository validationResultRepository; - public DataBaseProcessor(ValidationJob validationJob, final ValidationJobRepository validationJobRepository, + public DataBaseProcessor(ValidationJob validationJob, final ValidationResultRepository validationResultRepository){ this.validationJob = validationJob; - this.validationJobRepository = validationJobRepository; this.validationResultRepository = validationResultRepository; } @@ -26,7 +23,6 @@ public class DataBaseProcessor implements Processor { validationJob.progress = "COMPLETED"; validationJob.endDate = new Date(); validationJob.status = validationResultRepository.getStatus(validationJob.id); - System.out.println(validationJob.status); - validationJobRepository.save(validationJob); + exchange.getIn().setBody(validationJob, ValidationJob.class); } } diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java index 05758ce..961e960 100644 --- a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java +++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java @@ -6,8 +6,6 @@ import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfi import eu.dnetlib.validatorapi.entities.ValidationIssue; import eu.dnetlib.validatorapi.entities.ValidationJob; import eu.dnetlib.validatorapi.entities.ValidationRuleResult; -import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; -import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.w3c.dom.Document; @@ -18,13 +16,15 @@ import org.xml.sax.InputSource; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; public class XmlProcessor implements Processor { private final AbstractOpenAireProfile profile; private ValidationJob validationJob; - private final ValidationIssueRepository validationIssueRepository; - private final ValidationResultRepository validationResultRepository; + // private final ValidationIssueRepository validationIssueRepository; private final long maxNumberOfRecords; private int processedRecords; // private int resultSum; @@ -32,14 +32,10 @@ public class XmlProcessor implements Processor { public XmlProcessor(final AbstractOpenAireProfile profile, ValidationJob validationJob, - final ValidationIssueRepository validationIssueRepository, - final ValidationResultRepository validationResultRepository, final long maxNumberOfRecords){ super(); this.profile = profile; this.validationJob = validationJob; - this.validationIssueRepository = validationIssueRepository; - this.validationResultRepository = validationResultRepository; this.maxNumberOfRecords = maxNumberOfRecords; } @@ -51,6 +47,8 @@ public class XmlProcessor implements Processor { final DocumentBuilder documentBuilder = dbf.newDocumentBuilder(); Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml))); String recordUrl = extractRecordUrl(doc, "identifier"); + List validationRuleResults = new ArrayList<>(); + List validationIssues = new ArrayList<>(); if (profile != null) { XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that? @@ -61,49 +59,45 @@ public class XmlProcessor implements Processor { final Guideline.Result engineResult = (Guideline.Result) entry.getValue(); String ruleName = entry.getKey().toString(); validationRuleResult = new ValidationRuleResult(); - constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl, + constructValidationRuleResult(validationRuleResult, validationJob.id, profile.name(), recordUrl, ruleName, profile, engineResult); if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) { validationRuleResult.hasErrors = true; - saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR"); + for (String error:engineResult.errors()) { + validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, error,"ERROR")); + } } if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) { validationRuleResult.hasWarnings = true; - saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING"); + for (String warning:engineResult.warnings()) { + validationIssues.add(new ValidationIssue(validationJob.id, recordUrl, ruleName, warning, "WARNING")); + } } - validationResultRepository.save(validationRuleResult); + //validationResultRepository.save(validationRuleResult); + validationRuleResults.add(validationRuleResult); scoreSum += engineResult.score(); - } + } processedRecords++; validationJob.recordsTested = processedRecords; validationJob.score = scoreSum/processedRecords; - //System.out.println("\n\n\nvalidation job: " + validationJob.recordsTested); - if (processedRecords > maxNumberOfRecords) { + Map resultsMap = new HashMap(); + resultsMap.put("results", validationRuleResults); + resultsMap.put("issues", validationIssues); + exchange.getIn().setBody(resultsMap, Map.class); + + if (processedRecords == maxNumberOfRecords) { exchange.getIn().setHeader("MyHeader", "stop"); } } - private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, - Iterable issues, String issueType) { - for (String issue:issues) { - ValidationIssue validationIssue = new ValidationIssue(); - validationIssue.validationJobId = validationJobId; - validationIssue.ruleName = ruleName; - validationIssue.recordUrl = recordUrl; - validationIssue.issueType = issueType; - validationIssue.issueText = issue; - validationIssueRepository.save(validationIssue); - } - } - //TODO consider throwing exception - UTIL class? private String extractRecordUrl(final Document document, final String xmlField) { NodeList identifierNodes = document.getElementsByTagName(xmlField); @@ -115,10 +109,11 @@ public class XmlProcessor implements Processor { } - private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String recordUrl, String ruleName, + private void constructValidationRuleResult(ValidationRuleResult validationRuleResult, int validationJobId, String guidelines, String recordUrl, String ruleName, AbstractOpenAireProfile profile, Guideline.Result engineResult) { validationRuleResult.validationJobId = validationJobId; + validationRuleResult.guidelines = guidelines; validationRuleResult.ruleName = ruleName; validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight(); validationRuleResult.recordUrl = recordUrl; diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java index 4713d31..7237eb1 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java @@ -1,7 +1,9 @@ package eu.dnetlib.validatorapi.routes; import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile; +import eu.dnetlib.validatorapi.entities.ValidationIssue; import eu.dnetlib.validatorapi.entities.ValidationJob; +import eu.dnetlib.validatorapi.entities.ValidationRuleResult; import eu.dnetlib.validatorapi.processors.DataBaseProcessor; import eu.dnetlib.validatorapi.processors.XmlProcessor; import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; @@ -49,14 +51,24 @@ public class OaiPmhRoute extends RouteBuilder { from(oaiEndpoint) .routeId(date) .split(xpath("//*[local-name()='record']")) - .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository, maxNumberOfRecords)) - .choice().when(header("MyHeader").isEqualTo("stop")) - .to("direct:saveToDatabase") - .to("controlbus:route?routeId="+date+"&action=stop") + .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords)) + .choice() + .when(simple("${body[results]} && ${header.MyHeader} != 'stop'")) + .split(simple("${body[results]}")) + .to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true") + .endChoice() + .when(simple("${body[issues]} && ${header.MyHeader} != 'stop'")) + .split(simple("${body[issues]}")) + .to("jpa:"+ ValidationIssue.class.getName()+ "?usePersist=true") + .endChoice() + .when(header("MyHeader").isEqualTo("stop")) + //.to("direct:saveToDatabase") + .to("controlbus:route?routeId="+date+"&action=stop") .endChoice(); from("direct:saveToDatabase") - .process(new DataBaseProcessor(validationJob, validationJobRepository, validationResultRepository)); + .process(new DataBaseProcessor(validationJob, validationResultRepository)) + .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true"); /*from("timer://myTimer?fixedRate=true&period=60000") .routeId("OAIProcessingRoute") diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java index a0a1ef8..d30ab76 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java @@ -25,35 +25,24 @@ import java.util.List; @Component public class OaiSetListRoute extends RouteBuilder { - /*private final String oaiEndpoint;*/ - - public String xml = new String(); - - - /*public OaiSetListRoute(final String oaiEndpoint) { - this.oaiEndpoint = oaiEndpoint; - }*/ - - @Override public void configure() throws Exception { from("direct:getResponse") .process(exchange -> { - // Get the endpoint URL from the request parameter String endpoint = exchange.getIn().getHeader("endpoint", String.class); - // Set the dynamic endpoint URL in the exchange header exchange.getIn().setHeader("dynamicEndpoint", endpoint); }) .recipientList(header("dynamicEndpoint")) .process(exchange -> { List ListSetsXml = exchange.getIn().getBody(ArrayList.class); List desiredJsonParts = new ArrayList<>(); - String xmlSets =""; + String xmlSets = ""; + // Extract the desired elements from each XML string for (String xml : ListSetsXml) { List extractedSets = extractSets(xml); - for (String setXml:extractedSets) + for (String setXml : extractedSets) xmlSets += setXml; } @@ -87,7 +76,7 @@ public class OaiSetListRoute extends RouteBuilder { exchange.getIn().setBody(jsonString); }).to("seda:result");*/ //} - + private List extractSets(String xml) { List setXmls = new ArrayList<>(); try {