MaDgIK
/
uoa-validator-api
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

getSets method returns timeout if the process of getting all the sets from the oai server is not finished within 20 seconds.

This commit is contained in:
Katerina 2023-11-20 14:36:46 +02:00
parent 59fdd724e9
commit 88a57a8b85
4 changed files with 126 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
View File

@ -14,6 +14,7 @@ import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
import eu.dnetlib.validatorapi.routes.FairOaiPmhRoute2;
import eu.dnetlib.validatorapi.routes.SimpleOaiPmhRoute;
import org.apache.camel.CamelContext;
import org.apache.camel.Exchange;
import org.apache.camel.ProducerTemplate;
import org.apache.camel.builder.RouteBuilder;
import org.apache.http.HttpStatus;
@ -150,7 +151,6 @@ public class ValidationController {
//System.out.println("Final validation job "+ validationJob.hashCode());
validationJob.recordsTested = record;
validationJob.score = resultSum / record;
//TODO uncomment
log.info("Saving validation job " + validationJob.recordsTested);
validationJobRepository.save(validationJob);
}
@ -225,11 +225,14 @@ public class ValidationController {
if (fairProfile == null) {
oaiPmhRouteBuilder =
new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString(),
validationResultRepository, summaryValidationJobRepository);
validationJobRepository, validationResultRepository, summaryValidationJobRepository);
// new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
} else {
oaiPmhRouteBuilder =
new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
/* new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString(),
validationJobRepository, validationResultRepository, summaryValidationJobRepository);*/
}
@ -258,14 +261,20 @@ public class ValidationController {
camelContext.addRoutes(oaiSetListRoute);
}*/
@RequestMapping(value = {"/getSets"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public String getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
@RequestMapping(value = {"/getSets"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<String> getSets(@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL //not in use now
) throws Exception {
ProducerTemplate producerTemplate = camelContext.createProducerTemplate();
String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);
/*String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);*/
Exchange message = producerTemplate.request("direct:getResponse", exchange -> {
exchange.getMessage().setHeader("endpoint", "oaipmh://" + baseURL + "?verb=ListSets");
});
return response;
return ResponseEntity
//TODO: maybe something better if header is not set (null and mot 200)
.status(message.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, 200, Integer.class))
.body(message.getIn().getBody(String.class));
/*git git
System.out.println(camelContext.getRoutes());

90
src/main/java/eu/dnetlib/validatorapi/processors/SetsProcessor.java Normal file
View File

@ -0,0 +1,90 @@
package eu.dnetlib.validatorapi.processors;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.apache.camel.support.DefaultMessage;
import org.apache.http.HttpStatus;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.json.XML;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeoutException;
public class SetsProcessor implements Processor {
private final Logger log = LogManager.getLogger(this.getClass());
@Override
public void process(Exchange exchange) throws Exception {
List<DefaultMessage> messages = exchange.getIn().getBody(ArrayList.class);
//If the body is null a timeout has occurred.
if (exchange.getIn().getBody() == null)
throw new TimeoutException();
List<String> ListSetsXml = exchange.getIn().getBody(ArrayList.class);
List<String> desiredJsonParts = new ArrayList<>();
String xmlSets = "";
// Extract the desired <set> elements from each XML string
for (String xml : ListSetsXml) {
List<String> extractedSets = extractSets(xml);
for (String setXml : extractedSets)
xmlSets += setXml;
}
JSONObject jsonObject = XML.toJSONObject(xmlSets);
String jsonString = jsonObject.toString();
exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, HttpStatus.SC_OK);
exchange.getIn().setBody(jsonString);
}
private List<String> extractSets(String xml) {
List<String> setXmls = new ArrayList<>();
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = factory.newDocumentBuilder();
InputSource inputSource = new InputSource(new StringReader(xml));
Document document = builder.parse(inputSource);
NodeList setElements = document.getElementsByTagName("set");
for (int i = 0; i < setElements.getLength(); i++) {
Node setElement = setElements.item(i);
String setXml = nodeToString(setElement);
setXmls.add(setXml);
}
} catch (Exception e) {
// TODO: consider if Ineed to handle any exceptions that occur during
// XML parsing or processing and not only log them
log.error("Could not parse the xml ", e);
}
return setXmls;
}
private String nodeToString(Node node) throws TransformerException {
StringWriter sw = new StringWriter();
TransformerFactory tf = TransformerFactory.newInstance();
Transformer transformer = tf.newTransformer();
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
transformer.transform(new DOMSource(node), new StreamResult(sw));
return sw.toString();
}
}

2
src/main/java/eu/dnetlib/validatorapi/processors/SumUpNCleanProcessor.java Normal file
View File

@ -0,0 +1,2 @@
package eu.dnetlib.validatorapi.processors;public class SumUpNCleanProcessor {
}

114
src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java
View File

@ -1,32 +1,21 @@
package eu.dnetlib.validatorapi.routes;
import eu.dnetlib.validatorapi.processors.SetsProcessor;
import org.apache.camel.Exchange;
import org.apache.camel.TypeConversionException;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.engine.DefaultShutdownStrategy;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.StringReader;
import java.io.StringWriter;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeoutException;
@Component
public class OaiSetListRoute extends RouteBuilder {
@Override
public void configure() throws Exception {
@ -56,88 +45,23 @@ public class OaiSetListRoute extends RouteBuilder {
.handled(true)
.end();
from("direct:getResponse")
.log("\n\n\n LIST ")
.end();
onException(TimeoutException.class)
.handled(true)
.process(exchange -> {
exchange.getMessage().setHeader(Exchange.HTTP_RESPONSE_CODE, HttpStatus.SC_REQUEST_TIMEOUT);
exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, "application/json");
exchange.getIn().setBody("{\"error_code\":\"408\",\"message\":\"Unable to retrieve sets. Server response takes too long.\"}");
});
/*from("direct:getResponse")
from("direct:getResponse")
.routeId("myRoute")
.process(exchange -> {
String endpoint = exchange.getIn().getHeader("endpoint", String.class);
exchange.getIn().setHeader("dynamicEndpoint", endpoint);
})
.recipientList(header("dynamicEndpoint"))
.process(exchange -> {
List<String> ListSetsXml = exchange.getIn().getBody(ArrayList.class);
List<String> desiredJsonParts = new ArrayList<>();
String xmlSets = "";
// Extract the desired <set> elements from each XML string
for (String xml : ListSetsXml) {
List<String> extractedSets = extractSets(xml);
for (String setXml : extractedSets)
xmlSets += setXml;
}
JSONObject jsonObject = XML.toJSONObject(xmlSets);
String jsonString = jsonObject.toString();
exchange.getIn().setBody(jsonString);
})
.end();*/
}
/*from("direct:processSets")
.process(exchange -> {
// Access the individual response from the ArrayList
ArrayList<?> responseList = exchange.getIn().getBody(ArrayList.class);
InputStream responseStream = new ByteArrayInputStream(responseList.get(0).toString().getBytes());
// Set the converted InputStream as the new body
exchange.getIn().setBody(responseStream);
}).log("${body}")
.split(xpath("//*[local-name()='set']")).log("${body}")
// .log("${body}")
.aggregate(constant(true), new ListSetAggregationStrategy())
.completionPredicate(exchangeProperty(Exchange.AGGREGATED_SIZE).isEqualTo(header("CamelSplitSize")))
.to("direct:processAggregatedSets")
.log("${body}");*/
/*
from("direct:processAggregatedSets")
.process(exchange -> {
String aggregatedXml = exchange.getIn().getBody(ArrayList.class) + "";
JSONObject jsonObject = XML.toJSONObject(aggregatedXml);
String jsonString = jsonObject.toString();
exchange.getIn().setBody(jsonString);
}).to("seda:result");*/
//}
private List<String> extractSets(String xml) {
List<String> setXmls = new ArrayList<>();
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = factory.newDocumentBuilder();
InputSource inputSource = new InputSource(new StringReader(xml));
Document document = builder.parse(inputSource);
NodeList setElements = document.getElementsByTagName("set");
for (int i = 0; i < setElements.getLength(); i++) {
Node setElement = setElements.item(i);
String setXml = nodeToString(setElement);
setXmls.add(setXml);
}
} catch (Exception e) {
// Handle any exceptions that occur during XML parsing or processing
log.error(e.getMessage());
}
return setXmls;
}
private String nodeToString(Node node) throws TransformerException {
StringWriter sw = new StringWriter();
TransformerFactory tf = TransformerFactory.newInstance();
Transformer transformer = tf.newTransformer();
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
transformer.transform(new DOMSource(node), new StreamResult(sw));
return sw.toString();
}).recipientList(header("dynamicEndpoint"))
.parallelProcessing()
.timeout(20000)
.process(new SetsProcessor())
.end();
}
}