From 88a57a8b859ad5bab75df673971741653e84d50c Mon Sep 17 00:00:00 2001 From: Katerina Date: Mon, 20 Nov 2023 14:36:46 +0200 Subject: [PATCH] getSets method returns timeout if the process of getting all the sets from the oai server is not finished within 20 seconds. --- .../controllers/ValidationController.java | 21 +++- .../processors/SetsProcessor.java | 90 ++++++++++++++ .../processors/SumUpNCleanProcessor.java | 2 + .../validatorapi/routes/OaiSetListRoute.java | 114 +++--------------- 4 files changed, 126 insertions(+), 101 deletions(-) create mode 100644 src/main/java/eu/dnetlib/validatorapi/processors/SetsProcessor.java create mode 100644 src/main/java/eu/dnetlib/validatorapi/processors/SumUpNCleanProcessor.java diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index 0520f85..d5274ac 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -14,6 +14,7 @@ import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import eu.dnetlib.validatorapi.routes.FairOaiPmhRoute2; import eu.dnetlib.validatorapi.routes.SimpleOaiPmhRoute; import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; import org.apache.camel.builder.RouteBuilder; import org.apache.http.HttpStatus; @@ -150,7 +151,6 @@ public class ValidationController { //System.out.println("Final validation job "+ validationJob.hashCode()); validationJob.recordsTested = record; validationJob.score = resultSum / record; - //TODO uncomment log.info("Saving validation job " + validationJob.recordsTested); validationJobRepository.save(validationJob); } @@ -225,11 +225,14 @@ public class ValidationController { if (fairProfile == null) { oaiPmhRouteBuilder = new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString(), - validationResultRepository, summaryValidationJobRepository); + validationJobRepository, validationResultRepository, summaryValidationJobRepository); + // new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString()); } else { oaiPmhRouteBuilder = new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString()); + /* new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString(), + validationJobRepository, validationResultRepository, summaryValidationJobRepository);*/ } @@ -258,14 +261,20 @@ public class ValidationController { camelContext.addRoutes(oaiSetListRoute); }*/ - @RequestMapping(value = {"/getSets"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public String getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now + @RequestMapping(value = {"/getSets"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now ) throws Exception { ProducerTemplate producerTemplate = camelContext.createProducerTemplate(); - String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class); + /*String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);*/ + Exchange message = producerTemplate.request("direct:getResponse", exchange -> { + exchange.getMessage().setHeader("endpoint", "oaipmh://" + baseURL + "?verb=ListSets"); + }); - return response; + return ResponseEntity + //TODO: maybe something better if header is not set (null and mot 200) + .status(message.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, 200, Integer.class)) + .body(message.getIn().getBody(String.class)); /*git git System.out.println(camelContext.getRoutes()); diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/SetsProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/SetsProcessor.java new file mode 100644 index 0000000..a86716e --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/processors/SetsProcessor.java @@ -0,0 +1,90 @@ +package eu.dnetlib.validatorapi.processors; + +import org.apache.camel.Exchange; +import org.apache.camel.Processor; +import org.apache.camel.support.DefaultMessage; +import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import org.json.XML; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeoutException; + +public class SetsProcessor implements Processor { + + private final Logger log = LogManager.getLogger(this.getClass()); + + @Override + public void process(Exchange exchange) throws Exception { + List messages = exchange.getIn().getBody(ArrayList.class); + + //If the body is null a timeout has occurred. + if (exchange.getIn().getBody() == null) + throw new TimeoutException(); + + List ListSetsXml = exchange.getIn().getBody(ArrayList.class); + List desiredJsonParts = new ArrayList<>(); + String xmlSets = ""; + + // Extract the desired elements from each XML string + for (String xml : ListSetsXml) { + List extractedSets = extractSets(xml); + for (String setXml : extractedSets) + xmlSets += setXml; + } + + JSONObject jsonObject = XML.toJSONObject(xmlSets); + String jsonString = jsonObject.toString(); + exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, HttpStatus.SC_OK); + exchange.getIn().setBody(jsonString); + } + + private List extractSets(String xml) { + List setXmls = new ArrayList<>(); + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + InputSource inputSource = new InputSource(new StringReader(xml)); + Document document = builder.parse(inputSource); + + NodeList setElements = document.getElementsByTagName("set"); + for (int i = 0; i < setElements.getLength(); i++) { + Node setElement = setElements.item(i); + String setXml = nodeToString(setElement); + setXmls.add(setXml); + } + } catch (Exception e) { + // TODO: consider if Ineed to handle any exceptions that occur during + // XML parsing or processing and not only log them + log.error("Could not parse the xml ", e); + } + + return setXmls; + } + + private String nodeToString(Node node) throws TransformerException { + StringWriter sw = new StringWriter(); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.transform(new DOMSource(node), new StreamResult(sw)); + return sw.toString(); + } +} diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/SumUpNCleanProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/SumUpNCleanProcessor.java new file mode 100644 index 0000000..9d1c08c --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/processors/SumUpNCleanProcessor.java @@ -0,0 +1,2 @@ +package eu.dnetlib.validatorapi.processors;public class SumUpNCleanProcessor { +} diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java index becc05a..61ec357 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java @@ -1,32 +1,21 @@ package eu.dnetlib.validatorapi.routes; +import eu.dnetlib.validatorapi.processors.SetsProcessor; +import org.apache.camel.Exchange; import org.apache.camel.TypeConversionException; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.impl.engine.DefaultShutdownStrategy; +import org.apache.http.HttpStatus; import org.apache.http.client.ClientProtocolException; import org.springframework.stereotype.Component; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.StringReader; -import java.io.StringWriter; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.List; +import java.util.concurrent.TimeoutException; @Component public class OaiSetListRoute extends RouteBuilder { + @Override public void configure() throws Exception { @@ -56,88 +45,23 @@ public class OaiSetListRoute extends RouteBuilder { .handled(true) .end(); - from("direct:getResponse") - .log("\n\n\n LIST ") - .end(); + onException(TimeoutException.class) + .handled(true) + .process(exchange -> { + exchange.getMessage().setHeader(Exchange.HTTP_RESPONSE_CODE, HttpStatus.SC_REQUEST_TIMEOUT); + exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, "application/json"); + exchange.getIn().setBody("{\"error_code\":\"408\",\"message\":\"Unable to retrieve sets. Server response takes too long.\"}"); + }); - /*from("direct:getResponse") + from("direct:getResponse") + .routeId("myRoute") .process(exchange -> { String endpoint = exchange.getIn().getHeader("endpoint", String.class); exchange.getIn().setHeader("dynamicEndpoint", endpoint); - }) - .recipientList(header("dynamicEndpoint")) - .process(exchange -> { - List ListSetsXml = exchange.getIn().getBody(ArrayList.class); - List desiredJsonParts = new ArrayList<>(); - String xmlSets = ""; - - // Extract the desired elements from each XML string - for (String xml : ListSetsXml) { - List extractedSets = extractSets(xml); - for (String setXml : extractedSets) - xmlSets += setXml; - } - - JSONObject jsonObject = XML.toJSONObject(xmlSets); - String jsonString = jsonObject.toString(); - exchange.getIn().setBody(jsonString); - }) - .end();*/ -} - - /*from("direct:processSets") - .process(exchange -> { - // Access the individual response from the ArrayList - ArrayList responseList = exchange.getIn().getBody(ArrayList.class); - InputStream responseStream = new ByteArrayInputStream(responseList.get(0).toString().getBytes()); - - // Set the converted InputStream as the new body - exchange.getIn().setBody(responseStream); - }).log("${body}") - .split(xpath("//*[local-name()='set']")).log("${body}") - // .log("${body}") - .aggregate(constant(true), new ListSetAggregationStrategy()) - .completionPredicate(exchangeProperty(Exchange.AGGREGATED_SIZE).isEqualTo(header("CamelSplitSize"))) - .to("direct:processAggregatedSets") - .log("${body}");*/ -/* - from("direct:processAggregatedSets") - .process(exchange -> { - String aggregatedXml = exchange.getIn().getBody(ArrayList.class) + ""; - JSONObject jsonObject = XML.toJSONObject(aggregatedXml); - String jsonString = jsonObject.toString(); - exchange.getIn().setBody(jsonString); - }).to("seda:result");*/ - //} - - private List extractSets(String xml) { - List setXmls = new ArrayList<>(); - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - InputSource inputSource = new InputSource(new StringReader(xml)); - Document document = builder.parse(inputSource); - - NodeList setElements = document.getElementsByTagName("set"); - for (int i = 0; i < setElements.getLength(); i++) { - Node setElement = setElements.item(i); - String setXml = nodeToString(setElement); - setXmls.add(setXml); - } - } catch (Exception e) { - // Handle any exceptions that occur during XML parsing or processing - log.error(e.getMessage()); - } - - return setXmls; - } - - private String nodeToString(Node node) throws TransformerException { - StringWriter sw = new StringWriter(); - TransformerFactory tf = TransformerFactory.newInstance(); - Transformer transformer = tf.newTransformer(); - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); - transformer.transform(new DOMSource(node), new StreamResult(sw)); - return sw.toString(); + }).recipientList(header("dynamicEndpoint")) + .parallelProcessing() + .timeout(20000) + .process(new SetsProcessor()) + .end(); } }