From 50cb62af1eb8efa2fe39d31d68520ca3847a0176 Mon Sep 17 00:00:00 2001 From: Katerina Date: Tue, 19 Sep 2023 15:48:40 +0300 Subject: [PATCH] Added ExceptionProcessor for ClientProtocolException + TypeConversionException + Controller Level Exceptions --- .../controllers/ValidationController.java | 5 +- .../processors/ExceptionProcessor.java | 37 ++++ .../validatorapi/processors/XmlProcessor.java | 27 ++- .../validatorapi/routes/FairOaiPmhRoute2.java | 22 ++- .../routes/OAI_PMH_RouteBuilder.java | 185 +++++++++++++++++- .../validatorapi/routes/OaiSetListRoute.java | 7 + .../routes/SimpleOaiPmhRoute.java | 26 +++ 7 files changed, 295 insertions(+), 14 deletions(-) create mode 100644 src/main/java/eu/dnetlib/validatorapi/processors/ExceptionProcessor.java diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index ec95c9c..ceac486 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -226,8 +226,6 @@ public class ValidationController { oaiPmhRouteBuilder = new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString()); } else { - System.out.println("THERE"); - oaiPmhRouteBuilder = new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString()); } @@ -394,7 +392,8 @@ public class ValidationController { private String initializeMetadataPrefix(String guidelinesProfileName) { - if (guidelinesProfileName.equals("OpenAIRE Guidelines for Data Archives Profile v2")) { + if (guidelinesProfileName.equals("OpenAIRE Guidelines for Data Archives Profile v2") || + guidelinesProfileName.equals("OpenAIRE FAIR Guidelines for Data Repositories Profile")) { return "oai_datacite"; } else if (guidelinesProfileName.equals("OpenAIRE Guidelines for Literature Repositories Profile v3")) { diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/ExceptionProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/ExceptionProcessor.java new file mode 100644 index 0000000..9362990 --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/processors/ExceptionProcessor.java @@ -0,0 +1,37 @@ +package eu.dnetlib.validatorapi.processors; + +import eu.dnetlib.validatorapi.entities.ValidationJob; +import org.apache.camel.Exchange; +import org.apache.camel.Processor; + +import java.util.Date; + +public class ExceptionProcessor implements Processor { + + ValidationJob validationJob; + + public ExceptionProcessor(ValidationJob validationJob) { + this.validationJob = validationJob; + } + + @Override + public void process(Exchange exchange) { + Throwable caused = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Throwable.class); + if (caused.getClass() != null) { + if (caused.getClass().getName() != null) { + validationJob.exceptionClass = caused.getClass().getName(); + } + } + + if (caused.getCause() != null) { + if (caused.getCause().getMessage() != null) { + validationJob.exceptionMessage = caused.getCause().toString(); + } + } + + validationJob.progress = "STOPPED"; + validationJob.endDate = new Date(); + + exchange.getIn().setBody(validationJob, ValidationJob.class); + } +} diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java index 469fbce..5500e45 100644 --- a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java +++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java @@ -128,15 +128,24 @@ public class XmlProcessor implements Processor { validationRuleResult.validationJobId = validationJobId; validationRuleResult.guidelines = guidelines; validationRuleResult.ruleName = ruleName; - validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight(); - validationRuleResult.requirement_level = profile.guideline(ruleName).getRequirementLevel().name(); - System.out.println(validationRuleResult.requirement_level); - validationRuleResult.description = profile.guideline(ruleName).getDescription(); - System.out.println(validationRuleResult.description); - validationRuleResult.fair_principles = profile.guideline(ruleName).getFairPrinciples(); - System.out.println(validationRuleResult.fair_principles); - validationRuleResult.link = profile.guideline(ruleName).getLink(); - System.out.println(validationRuleResult.link); + + if (profile.guideline(ruleName) != null) { + validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight(); + + if (profile.guideline(ruleName).getRequirementLevel() != null + && profile.guideline(ruleName).getRequirementLevel().name()!=null) + validationRuleResult.requirement_level = profile.guideline(ruleName).getRequirementLevel().name(); + + if (profile.guideline(ruleName).getDescription() != null) + validationRuleResult.description = profile.guideline(ruleName).getDescription(); + + if (profile.guideline(ruleName).getFairPrinciples() != null) + validationRuleResult.fair_principles = profile.guideline(ruleName).getFairPrinciples(); + + if (profile.guideline(ruleName).getLink() != null) + validationRuleResult.link = profile.guideline(ruleName).getLink(); + } + validationRuleResult.recordUrl = recordUrl; validationRuleResult.score = engineResult.score(); validationRuleResult.status = engineResult.status().toString(); diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/FairOaiPmhRoute2.java b/src/main/java/eu/dnetlib/validatorapi/routes/FairOaiPmhRoute2.java index d49b672..ff2a28e 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/FairOaiPmhRoute2.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/FairOaiPmhRoute2.java @@ -6,10 +6,13 @@ import eu.dnetlib.validatorapi.entities.ValidationIssue; import eu.dnetlib.validatorapi.entities.ValidationJob; import eu.dnetlib.validatorapi.entities.ValidationRuleResult; import eu.dnetlib.validatorapi.processors.DataBaseProcessor; +import eu.dnetlib.validatorapi.processors.ExceptionProcessor; import eu.dnetlib.validatorapi.processors.XmlProcessor; import org.apache.camel.Exchange; import org.apache.camel.Processor; +import org.apache.camel.TypeConversionException; import org.apache.camel.builder.RouteBuilder; +import org.apache.http.client.ClientProtocolException; import java.util.Date; import java.util.UUID; @@ -37,9 +40,26 @@ public class FairOaiPmhRoute2 extends RouteBuilder { @Override public void configure() throws Exception { - String date = new Date().toString(); + onException(TypeConversionException.class) + .process(new ExceptionProcessor(validationJob)) + .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true").log("\n\n\n\nHEREEEEEE") + .to("controlbus:route?routeId="+routeid+"&action=stop") + .to("controlbus:route?routeId="+routeid2+"&action=stop") + .maximumRedeliveries(0) + .handled(true) + .end(); + + onException(ClientProtocolException.class) + .process(new ExceptionProcessor(validationJob)) + .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true") + .to("controlbus:route?routeId="+routeid+"&action=stop") + .to("controlbus:route?routeId="+routeid2+"&action=stop") + .maximumRedeliveries(0) + .handled(true) + .end(); + from(oaiEndpoint) .multicast().parallelProcessing() .to("direct:guidelinesProcessor") diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OAI_PMH_RouteBuilder.java b/src/main/java/eu/dnetlib/validatorapi/routes/OAI_PMH_RouteBuilder.java index 3783bd5..284742b 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OAI_PMH_RouteBuilder.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OAI_PMH_RouteBuilder.java @@ -1,2 +1,185 @@ -package eu.dnetlib.validatorapi.routes;public class OAI_PMH_RouteBuilder { +package eu.dnetlib.validatorapi.routes; + +import eu.dnetlib.validator2.validation.guideline.openaire.LiteratureGuidelinesV4Profile; +import eu.dnetlib.validatorapi.entities.ValidationIssue; +import eu.dnetlib.validatorapi.entities.ValidationJob; +import eu.dnetlib.validatorapi.entities.ValidationRuleResult; +import eu.dnetlib.validatorapi.processors.XmlProcessor; +import org.apache.camel.builder.RouteBuilder; +import org.springframework.stereotype.Component; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; + +@Component +public class OAI_PMH_RouteBuilder extends RouteBuilder { +/* + private final String oaiEndpoint = "oaipmh://http://repositorium.sdum.uminho.pt/oai/request"; + private AbstractOpenAireProfile profile; + private long maxNumberOfRecords = 0; + private ValidationJob validationJob; + + private final ValidationJobRepository validationJobRepository; + private final ValidationIssueRepository validationIssueRepository; + private final ValidationResultRepository validationResultRepository; + + public OAI_PMH_RouteBuilder(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob, + long maxNumberOfRecords, final ValidationJobRepository validationJobRepository, + final ValidationIssueRepository validationIssueRepository, + final ValidationResultRepository validationResultRepository) { + this.validationJob = validationJob; + this.profile = profile; + this.maxNumberOfRecords = maxNumberOfRecords; + this.validationJobRepository = validationJobRepository; + this.validationIssueRepository = validationIssueRepository; + this.validationResultRepository = validationResultRepository; + }*/ + + @Override + public void configure() throws Exception { + + /* String id1 = new Date().getTime()+""; + String id2 = new Date().getTime()+1+""; + String date = new Date().toString(); +*/ + from("direct:startProcessing") + .toD("${body}") // Use dynamic endpoint from the message body (the endpoint set by OaiPmhRoute) + .routeId("Process_OAI_Response") + .split(xpath("//*[local-name()='record']")) + .process(new XmlProcessor(new LiteratureGuidelinesV4Profile(), new ValidationJob(), 1)) + .choice() + .when(simple("${body[results]} && ${header.MyHeader} != 'stop'")) + .split(simple("${body[results]}")) + .to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true") + .stop() // Stop the route when this condition is true + .endChoice() + .end() + .choice() + .when(simple("${body[issues]} && ${header.MyHeader} != 'stop'")) + .split(simple("${body[issues]}")) + .to("jpa:" + ValidationIssue.class.getName() + "?usePersist=true") + .stop() // Stop the route when this condition is true + .endChoice() + .end() + .choice() + .when(header("MyHeader").isEqualTo("stop")) + .to("direct:saveToDatabase") + .to("controlbus:route?routeId=Process_OAI_Response&action=stop") + .endChoice() + .end(); + + + + + /*from("direct:getRecords") + .toD("${header.oaiEndpoint}") + .routeId(date) + .split(xpath("//*[local-name()='record']")) + .log("${body}") + .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords)) + .choice() + .when(simple("${body[issues]} && ${header.MyHeader} != 'stop'")) + .log("HERE!!!") + .split(simple("${body[issues]}")) + .to("jpa:"+ ValidationRuleResult.class.getName()+ "?usePersist=true") + .endChoice() + .end() + .choice() + .when(simple("${body[results]} && ${header.MyHeader} != 'stop'")) + .log("THERE") + .split(simple("${body[results]}")) + .to("jpa:" + ValidationIssue.class.getName() + "?usePersist=true") + .endChoice() + .end() + .choice() + .when(header("MyHeader").isEqualTo("stop")) + .to("direct:saveToDatabase") + .to("controlbus:route?routeId="+date+"&action=stop") + .endChoice() + .end(); + + + from("direct:saveToDatabase") + .process(new DataBaseProcessor()) + .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true");*/ + + /* + from(oaiEndpoint) + .multicast().parallelProcessing() + .to("direct:guidelinesProcessor") + .to("direct:fairProcessor") + .end(); + + from("direct:guidelinesProcessor") + .routeId(id1) + .split(xpath("//*[local-name()='record']")) + //TODO .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords)) + //.log("PROCESSOR 1111 \n\n\n ${body}") + .choice().when(header("MyHeader").isEqualTo("stop")) + //.to("direct:saveToDatabase") + .to("controlbus:route?routeId="+id1+"&action=stop") + .endChoice(); + + from("direct:fairProcessor") + .routeId(id2) + .split(xpath("//*[local-name()='record']")) + //.log("PROCESSOR 222 \n\n\n ${body}") + //TODO .process(new XmlProcessor(new FAIR_Data_GuidelinesProfile(), validationJob, maxNumberOfRecords)) + .choice().when(header("MyHeader").isEqualTo("stop")) + //.to("direct:saveToDatabase") + .to("controlbus:route?routeId="+id2+"&action=stop") + .endChoice(); + + + from("direct:saveToDatabase") //TODO FIX THIS with JPA + .process(new DataBaseProcessor()); + + + */ + } + + private List extractSets(String xml) { + List setXmls = new ArrayList<>(); + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + InputSource inputSource = new InputSource(new StringReader(xml)); + Document document = builder.parse(inputSource); + + NodeList setElements = document.getElementsByTagName("set"); + for (int i = 0; i < setElements.getLength(); i++) { + Node setElement = setElements.item(i); + String setXml = nodeToString(setElement); + setXmls.add(setXml); + } + } catch (Exception e) { + // Handle any exceptions that occur during XML parsing or processing + log.error(e.getMessage()); + } + + return setXmls; + } + + private String nodeToString(Node node) throws TransformerException { + StringWriter sw = new StringWriter(); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.transform(new DOMSource(node), new StreamResult(sw)); + return sw.toString(); + } } diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java index d30ab76..ea2ba79 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java @@ -1,6 +1,7 @@ package eu.dnetlib.validatorapi.routes; import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.impl.engine.DefaultShutdownStrategy; import org.json.JSONObject; import org.json.XML; import org.springframework.stereotype.Component; @@ -28,6 +29,12 @@ public class OaiSetListRoute extends RouteBuilder { @Override public void configure() throws Exception { + // Access the DefaultShutdownStrategy + DefaultShutdownStrategy shutdownStrategy = (DefaultShutdownStrategy) getContext().getShutdownStrategy(); + + // Set the shutdown timeout in milliseconds (e.g., 10 seconds) + shutdownStrategy.setTimeout(1); + from("direct:getResponse") .process(exchange -> { String endpoint = exchange.getIn().getHeader("endpoint", String.class); diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/SimpleOaiPmhRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/SimpleOaiPmhRoute.java index e17bb2e..114a8dc 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/SimpleOaiPmhRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/SimpleOaiPmhRoute.java @@ -5,10 +5,13 @@ import eu.dnetlib.validatorapi.entities.ValidationIssue; import eu.dnetlib.validatorapi.entities.ValidationJob; import eu.dnetlib.validatorapi.entities.ValidationRuleResult; import eu.dnetlib.validatorapi.processors.DataBaseProcessor; +import eu.dnetlib.validatorapi.processors.ExceptionProcessor; import eu.dnetlib.validatorapi.processors.XmlProcessor; import org.apache.camel.Exchange; import org.apache.camel.Processor; +import org.apache.camel.TypeConversionException; import org.apache.camel.builder.RouteBuilder; +import org.apache.http.client.ClientProtocolException; import java.util.Date; @@ -37,9 +40,32 @@ public class SimpleOaiPmhRoute extends RouteBuilder { String date = new Date().toString(); + onException(TypeConversionException.class) + .process(new ExceptionProcessor(validationJob)) + .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true").log("\n\n\n\nHEREEEEEE") + .to("controlbus:route?routeId="+routeid+"&action=stop") + .maximumRedeliveries(0) + .handled(true) + .end(); + + onException(ClientProtocolException.class) + .process(new ExceptionProcessor(validationJob)) + .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true") + .to("controlbus:route?routeId="+routeid+"&action=stop") + .maximumRedeliveries(0) + .handled(true) + .end(); + +/* + .process(new TypeConversionExceptionProcessor(validationJob)).log("\n\n\n\n HERE 2") + .process(new DataBaseProcessor())//.maximumRedeliveries(0) + .to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true").log("\n\n\n\nHEREEEEEE") + .stop(); +*/ from(oaiEndpoint) .routeId(routeid) .split(xpath("//*[local-name()='record']")) + .doTry() .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords)) .choice() .when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))