getSets method returns timeout if the process of getting all the sets from the oai server is not finished within 20 seconds.
This commit is contained in:
parent
59fdd724e9
commit
88a57a8b85
|
@ -14,6 +14,7 @@ import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
|
||||||
import eu.dnetlib.validatorapi.routes.FairOaiPmhRoute2;
|
import eu.dnetlib.validatorapi.routes.FairOaiPmhRoute2;
|
||||||
import eu.dnetlib.validatorapi.routes.SimpleOaiPmhRoute;
|
import eu.dnetlib.validatorapi.routes.SimpleOaiPmhRoute;
|
||||||
import org.apache.camel.CamelContext;
|
import org.apache.camel.CamelContext;
|
||||||
|
import org.apache.camel.Exchange;
|
||||||
import org.apache.camel.ProducerTemplate;
|
import org.apache.camel.ProducerTemplate;
|
||||||
import org.apache.camel.builder.RouteBuilder;
|
import org.apache.camel.builder.RouteBuilder;
|
||||||
import org.apache.http.HttpStatus;
|
import org.apache.http.HttpStatus;
|
||||||
|
@ -150,7 +151,6 @@ public class ValidationController {
|
||||||
//System.out.println("Final validation job "+ validationJob.hashCode());
|
//System.out.println("Final validation job "+ validationJob.hashCode());
|
||||||
validationJob.recordsTested = record;
|
validationJob.recordsTested = record;
|
||||||
validationJob.score = resultSum / record;
|
validationJob.score = resultSum / record;
|
||||||
//TODO uncomment
|
|
||||||
log.info("Saving validation job " + validationJob.recordsTested);
|
log.info("Saving validation job " + validationJob.recordsTested);
|
||||||
validationJobRepository.save(validationJob);
|
validationJobRepository.save(validationJob);
|
||||||
}
|
}
|
||||||
|
@ -225,11 +225,14 @@ public class ValidationController {
|
||||||
if (fairProfile == null) {
|
if (fairProfile == null) {
|
||||||
oaiPmhRouteBuilder =
|
oaiPmhRouteBuilder =
|
||||||
new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString(),
|
new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString(),
|
||||||
validationResultRepository, summaryValidationJobRepository);
|
validationJobRepository, validationResultRepository, summaryValidationJobRepository);
|
||||||
|
|
||||||
|
// new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
|
||||||
} else {
|
} else {
|
||||||
oaiPmhRouteBuilder =
|
oaiPmhRouteBuilder =
|
||||||
new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
|
new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
|
||||||
|
/* new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString(),
|
||||||
|
validationJobRepository, validationResultRepository, summaryValidationJobRepository);*/
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -258,14 +261,20 @@ public class ValidationController {
|
||||||
camelContext.addRoutes(oaiSetListRoute);
|
camelContext.addRoutes(oaiSetListRoute);
|
||||||
}*/
|
}*/
|
||||||
|
|
||||||
@RequestMapping(value = {"/getSets"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
|
@RequestMapping(value = {"/getSets"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||||
public String getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
|
public ResponseEntity<String> getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
|
||||||
) throws Exception {
|
) throws Exception {
|
||||||
|
|
||||||
ProducerTemplate producerTemplate = camelContext.createProducerTemplate();
|
ProducerTemplate producerTemplate = camelContext.createProducerTemplate();
|
||||||
String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);
|
/*String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);*/
|
||||||
|
Exchange message = producerTemplate.request("direct:getResponse", exchange -> {
|
||||||
|
exchange.getMessage().setHeader("endpoint", "oaipmh://" + baseURL + "?verb=ListSets");
|
||||||
|
});
|
||||||
|
|
||||||
return response;
|
return ResponseEntity
|
||||||
|
//TODO: maybe something better if header is not set (null and mot 200)
|
||||||
|
.status(message.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, 200, Integer.class))
|
||||||
|
.body(message.getIn().getBody(String.class));
|
||||||
|
|
||||||
/*git git
|
/*git git
|
||||||
System.out.println(camelContext.getRoutes());
|
System.out.println(camelContext.getRoutes());
|
||||||
|
|
|
@ -0,0 +1,90 @@
|
||||||
|
package eu.dnetlib.validatorapi.processors;
|
||||||
|
|
||||||
|
import org.apache.camel.Exchange;
|
||||||
|
import org.apache.camel.Processor;
|
||||||
|
import org.apache.camel.support.DefaultMessage;
|
||||||
|
import org.apache.http.HttpStatus;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.json.JSONObject;
|
||||||
|
import org.json.XML;
|
||||||
|
import org.w3c.dom.Document;
|
||||||
|
import org.w3c.dom.Node;
|
||||||
|
import org.w3c.dom.NodeList;
|
||||||
|
import org.xml.sax.InputSource;
|
||||||
|
|
||||||
|
import javax.xml.parsers.DocumentBuilder;
|
||||||
|
import javax.xml.parsers.DocumentBuilderFactory;
|
||||||
|
import javax.xml.transform.OutputKeys;
|
||||||
|
import javax.xml.transform.Transformer;
|
||||||
|
import javax.xml.transform.TransformerException;
|
||||||
|
import javax.xml.transform.TransformerFactory;
|
||||||
|
import javax.xml.transform.dom.DOMSource;
|
||||||
|
import javax.xml.transform.stream.StreamResult;
|
||||||
|
import java.io.StringReader;
|
||||||
|
import java.io.StringWriter;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.concurrent.TimeoutException;
|
||||||
|
|
||||||
|
public class SetsProcessor implements Processor {
|
||||||
|
|
||||||
|
private final Logger log = LogManager.getLogger(this.getClass());
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void process(Exchange exchange) throws Exception {
|
||||||
|
List<DefaultMessage> messages = exchange.getIn().getBody(ArrayList.class);
|
||||||
|
|
||||||
|
//If the body is null a timeout has occurred.
|
||||||
|
if (exchange.getIn().getBody() == null)
|
||||||
|
throw new TimeoutException();
|
||||||
|
|
||||||
|
List<String> ListSetsXml = exchange.getIn().getBody(ArrayList.class);
|
||||||
|
List<String> desiredJsonParts = new ArrayList<>();
|
||||||
|
String xmlSets = "";
|
||||||
|
|
||||||
|
// Extract the desired <set> elements from each XML string
|
||||||
|
for (String xml : ListSetsXml) {
|
||||||
|
List<String> extractedSets = extractSets(xml);
|
||||||
|
for (String setXml : extractedSets)
|
||||||
|
xmlSets += setXml;
|
||||||
|
}
|
||||||
|
|
||||||
|
JSONObject jsonObject = XML.toJSONObject(xmlSets);
|
||||||
|
String jsonString = jsonObject.toString();
|
||||||
|
exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, HttpStatus.SC_OK);
|
||||||
|
exchange.getIn().setBody(jsonString);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> extractSets(String xml) {
|
||||||
|
List<String> setXmls = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||||
|
DocumentBuilder builder = factory.newDocumentBuilder();
|
||||||
|
InputSource inputSource = new InputSource(new StringReader(xml));
|
||||||
|
Document document = builder.parse(inputSource);
|
||||||
|
|
||||||
|
NodeList setElements = document.getElementsByTagName("set");
|
||||||
|
for (int i = 0; i < setElements.getLength(); i++) {
|
||||||
|
Node setElement = setElements.item(i);
|
||||||
|
String setXml = nodeToString(setElement);
|
||||||
|
setXmls.add(setXml);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
// TODO: consider if Ineed to handle any exceptions that occur during
|
||||||
|
// XML parsing or processing and not only log them
|
||||||
|
log.error("Could not parse the xml ", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
return setXmls;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nodeToString(Node node) throws TransformerException {
|
||||||
|
StringWriter sw = new StringWriter();
|
||||||
|
TransformerFactory tf = TransformerFactory.newInstance();
|
||||||
|
Transformer transformer = tf.newTransformer();
|
||||||
|
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
|
||||||
|
transformer.transform(new DOMSource(node), new StreamResult(sw));
|
||||||
|
return sw.toString();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,2 @@
|
||||||
|
package eu.dnetlib.validatorapi.processors;public class SumUpNCleanProcessor {
|
||||||
|
}
|
|
@ -1,32 +1,21 @@
|
||||||
package eu.dnetlib.validatorapi.routes;
|
package eu.dnetlib.validatorapi.routes;
|
||||||
|
|
||||||
|
import eu.dnetlib.validatorapi.processors.SetsProcessor;
|
||||||
|
import org.apache.camel.Exchange;
|
||||||
import org.apache.camel.TypeConversionException;
|
import org.apache.camel.TypeConversionException;
|
||||||
import org.apache.camel.builder.RouteBuilder;
|
import org.apache.camel.builder.RouteBuilder;
|
||||||
import org.apache.camel.impl.engine.DefaultShutdownStrategy;
|
import org.apache.camel.impl.engine.DefaultShutdownStrategy;
|
||||||
|
import org.apache.http.HttpStatus;
|
||||||
import org.apache.http.client.ClientProtocolException;
|
import org.apache.http.client.ClientProtocolException;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.w3c.dom.Document;
|
|
||||||
import org.w3c.dom.Node;
|
|
||||||
import org.w3c.dom.NodeList;
|
|
||||||
import org.xml.sax.InputSource;
|
|
||||||
|
|
||||||
import javax.xml.parsers.DocumentBuilder;
|
|
||||||
import javax.xml.parsers.DocumentBuilderFactory;
|
|
||||||
import javax.xml.transform.OutputKeys;
|
|
||||||
import javax.xml.transform.Transformer;
|
|
||||||
import javax.xml.transform.TransformerException;
|
|
||||||
import javax.xml.transform.TransformerFactory;
|
|
||||||
import javax.xml.transform.dom.DOMSource;
|
|
||||||
import javax.xml.transform.stream.StreamResult;
|
|
||||||
import java.io.StringReader;
|
|
||||||
import java.io.StringWriter;
|
|
||||||
import java.net.UnknownHostException;
|
import java.net.UnknownHostException;
|
||||||
import java.util.ArrayList;
|
import java.util.concurrent.TimeoutException;
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class OaiSetListRoute extends RouteBuilder {
|
public class OaiSetListRoute extends RouteBuilder {
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void configure() throws Exception {
|
public void configure() throws Exception {
|
||||||
|
|
||||||
|
@ -56,88 +45,23 @@ public class OaiSetListRoute extends RouteBuilder {
|
||||||
.handled(true)
|
.handled(true)
|
||||||
.end();
|
.end();
|
||||||
|
|
||||||
from("direct:getResponse")
|
onException(TimeoutException.class)
|
||||||
.log("\n\n\n LIST ")
|
.handled(true)
|
||||||
.end();
|
.process(exchange -> {
|
||||||
|
exchange.getMessage().setHeader(Exchange.HTTP_RESPONSE_CODE, HttpStatus.SC_REQUEST_TIMEOUT);
|
||||||
|
exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, "application/json");
|
||||||
|
exchange.getIn().setBody("{\"error_code\":\"408\",\"message\":\"Unable to retrieve sets. Server response takes too long.\"}");
|
||||||
|
});
|
||||||
|
|
||||||
/*from("direct:getResponse")
|
from("direct:getResponse")
|
||||||
|
.routeId("myRoute")
|
||||||
.process(exchange -> {
|
.process(exchange -> {
|
||||||
String endpoint = exchange.getIn().getHeader("endpoint", String.class);
|
String endpoint = exchange.getIn().getHeader("endpoint", String.class);
|
||||||
exchange.getIn().setHeader("dynamicEndpoint", endpoint);
|
exchange.getIn().setHeader("dynamicEndpoint", endpoint);
|
||||||
})
|
}).recipientList(header("dynamicEndpoint"))
|
||||||
.recipientList(header("dynamicEndpoint"))
|
.parallelProcessing()
|
||||||
.process(exchange -> {
|
.timeout(20000)
|
||||||
List<String> ListSetsXml = exchange.getIn().getBody(ArrayList.class);
|
.process(new SetsProcessor())
|
||||||
List<String> desiredJsonParts = new ArrayList<>();
|
.end();
|
||||||
String xmlSets = "";
|
|
||||||
|
|
||||||
// Extract the desired <set> elements from each XML string
|
|
||||||
for (String xml : ListSetsXml) {
|
|
||||||
List<String> extractedSets = extractSets(xml);
|
|
||||||
for (String setXml : extractedSets)
|
|
||||||
xmlSets += setXml;
|
|
||||||
}
|
|
||||||
|
|
||||||
JSONObject jsonObject = XML.toJSONObject(xmlSets);
|
|
||||||
String jsonString = jsonObject.toString();
|
|
||||||
exchange.getIn().setBody(jsonString);
|
|
||||||
})
|
|
||||||
.end();*/
|
|
||||||
}
|
|
||||||
|
|
||||||
/*from("direct:processSets")
|
|
||||||
.process(exchange -> {
|
|
||||||
// Access the individual response from the ArrayList
|
|
||||||
ArrayList<?> responseList = exchange.getIn().getBody(ArrayList.class);
|
|
||||||
InputStream responseStream = new ByteArrayInputStream(responseList.get(0).toString().getBytes());
|
|
||||||
|
|
||||||
// Set the converted InputStream as the new body
|
|
||||||
exchange.getIn().setBody(responseStream);
|
|
||||||
}).log("${body}")
|
|
||||||
.split(xpath("//*[local-name()='set']")).log("${body}")
|
|
||||||
// .log("${body}")
|
|
||||||
.aggregate(constant(true), new ListSetAggregationStrategy())
|
|
||||||
.completionPredicate(exchangeProperty(Exchange.AGGREGATED_SIZE).isEqualTo(header("CamelSplitSize")))
|
|
||||||
.to("direct:processAggregatedSets")
|
|
||||||
.log("${body}");*/
|
|
||||||
/*
|
|
||||||
from("direct:processAggregatedSets")
|
|
||||||
.process(exchange -> {
|
|
||||||
String aggregatedXml = exchange.getIn().getBody(ArrayList.class) + "";
|
|
||||||
JSONObject jsonObject = XML.toJSONObject(aggregatedXml);
|
|
||||||
String jsonString = jsonObject.toString();
|
|
||||||
exchange.getIn().setBody(jsonString);
|
|
||||||
}).to("seda:result");*/
|
|
||||||
//}
|
|
||||||
|
|
||||||
private List<String> extractSets(String xml) {
|
|
||||||
List<String> setXmls = new ArrayList<>();
|
|
||||||
try {
|
|
||||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
|
||||||
DocumentBuilder builder = factory.newDocumentBuilder();
|
|
||||||
InputSource inputSource = new InputSource(new StringReader(xml));
|
|
||||||
Document document = builder.parse(inputSource);
|
|
||||||
|
|
||||||
NodeList setElements = document.getElementsByTagName("set");
|
|
||||||
for (int i = 0; i < setElements.getLength(); i++) {
|
|
||||||
Node setElement = setElements.item(i);
|
|
||||||
String setXml = nodeToString(setElement);
|
|
||||||
setXmls.add(setXml);
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
// Handle any exceptions that occur during XML parsing or processing
|
|
||||||
log.error(e.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
return setXmls;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String nodeToString(Node node) throws TransformerException {
|
|
||||||
StringWriter sw = new StringWriter();
|
|
||||||
TransformerFactory tf = TransformerFactory.newInstance();
|
|
||||||
Transformer transformer = tf.newTransformer();
|
|
||||||
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
|
|
||||||
transformer.transform(new DOMSource(node), new StreamResult(sw));
|
|
||||||
return sw.toString();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue