diff --git a/pom.xml b/pom.xml index 60106bb..eb0e8f1 100644 --- a/pom.xml +++ b/pom.xml @@ -89,40 +89,6 @@ camel-spring-boot-starter ${apache.camel.version} - - org.apache.camel.springboot - camel-jackson-starter - ${apache.camel.version} - - - org.apache.camel - camel-jackson - ${apache.camel.version} - - - - - org.apache.camel - camel-jacksonxml - ${apache.camel.version} - - - - com.fasterxml.jackson.core - jackson-core - 2.15.2 - - - com.fasterxml.jackson.core - jackson-databind - 2.15.2 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - 2.15.2 - - org.apache.camel.springboot camel-servlet-starter @@ -133,10 +99,17 @@ camel-http ${apache.camel.version} + + - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - 2.15.2 + org.json + json + 20230618 + + + javax.xml.bind + jaxb-api + 2.3.1 @@ -145,7 +118,7 @@ org.springframework.boot spring-boot-maven-plugin - 1.5.22.RELEASE + 2.7.10 eu.dnetlib.validatorapi.ValidatorApiApplication true diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index ae5ab95..35b770d 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -13,6 +13,7 @@ import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import eu.dnetlib.validatorapi.routes.OaiPmhRoute; import eu.dnetlib.validatorapi.routes.OaiSetListRoute; import org.apache.camel.CamelContext; +import org.apache.camel.ConsumerTemplate; import org.apache.camel.ProducerTemplate; import org.apache.camel.builder.RouteBuilder; import org.apache.logging.log4j.LogManager; @@ -81,7 +82,6 @@ public class ValidationController { log.error("Exception: No valid guidelines"); new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided."); } - System.out.println("AAAAAAAA"); ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords); validationJob.guidelines = profile.name(); validationJobRepository.save(validationJob); @@ -240,8 +240,11 @@ public class ValidationController { RouteBuilder listSetsBuilder = new OaiSetListRoute(); ProducerTemplate producerTemplate = camelContext.createProducerTemplate(); - String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class); - return response; + producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class); + + ConsumerTemplate consumerTemplate = camelContext.createConsumerTemplate(); + String result = consumerTemplate.receive("seda:result").getIn().getBody(String.class); + return result; /* System.out.println(camelContext.getRoutes()); diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java index ede8171..e9b2e9f 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java @@ -12,13 +12,16 @@ public class ValidationJob { @Column(name = "base_url") public String baseUrl; + @Column(name="number_of_records") public int numberOfRecords; + @Column(name="guidelines") public String guidelines; @Column(name="started") public Date startDate; + @Column(name="ended") public Date endDate; diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java index 0c3b154..05758ce 100644 --- a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java +++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java @@ -27,7 +27,8 @@ public class XmlProcessor implements Processor { private final ValidationResultRepository validationResultRepository; private final long maxNumberOfRecords; private int processedRecords; - private int resultSum; + // private int resultSum; + private int scoreSum; public XmlProcessor(final AbstractOpenAireProfile profile, ValidationJob validationJob, @@ -56,13 +57,14 @@ public class XmlProcessor implements Processor { Map results = validationResult.results(); ValidationRuleResult validationRuleResult; - for (Map.Entry entry : results.entrySet()) { + for (Map.Entry entry : results.entrySet()) { //for each rule of the record final Guideline.Result engineResult = (Guideline.Result) entry.getValue(); String ruleName = entry.getKey().toString(); validationRuleResult = new ValidationRuleResult(); constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl, ruleName, profile, engineResult); + if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) { validationRuleResult.hasErrors = true; saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR"); @@ -74,13 +76,14 @@ public class XmlProcessor implements Processor { } validationResultRepository.save(validationRuleResult); - resultSum += engineResult.score(); + scoreSum += engineResult.score(); + } } processedRecords++; validationJob.recordsTested = processedRecords; - validationJob.score = resultSum; + validationJob.score = scoreSum/processedRecords; //System.out.println("\n\n\nvalidation job: " + validationJob.recordsTested); if (processedRecords > maxNumberOfRecords) { diff --git a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java index f8864c4..9e60a47 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java @@ -28,7 +28,11 @@ public interface ValidationResultRepository extends JpaRepository 0 THEN TRUE ELSE FALSE END AS hasErrors," + "CASE WHEN COUNT(CASE WHEN has_warnings IS TRUE THEN 1 END) > 0 THEN TRUE ELSE FALSE END AS hasWarnings)" + diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java index 74cdf86..6095a55 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiSetListRoute.java @@ -3,7 +3,8 @@ package eu.dnetlib.validatorapi.routes; import eu.dnetlib.validatorapi.utils.ListSetAggregationStrategy; import org.apache.camel.Exchange; import org.apache.camel.builder.RouteBuilder; -import org.apache.camel.model.dataformat.JacksonXMLDataFormat; +import org.json.JSONObject; +import org.json.XML; import org.springframework.stereotype.Component; import java.io.ByteArrayInputStream; @@ -17,6 +18,7 @@ public class OaiSetListRoute extends RouteBuilder { public String xml = new String(); + /*public OaiSetListRoute(final String oaiEndpoint) { this.oaiEndpoint = oaiEndpoint; }*/ @@ -24,21 +26,18 @@ public class OaiSetListRoute extends RouteBuilder { @Override public void configure() throws Exception { - JacksonXMLDataFormat xmlJsonFormat = new JacksonXMLDataFormat(); - from("direct:getResponse") - .process(exchange -> { - // Get the endpoint URL from the request parameter - String endpoint = exchange.getIn().getHeader("endpoint", String.class); + .process(exchange -> { + // Get the endpoint URL from the request parameter + String endpoint = exchange.getIn().getHeader("endpoint", String.class); + // Set the dynamic endpoint URL in the exchange header + exchange.getIn().setHeader("dynamicEndpoint", endpoint); + }) + .recipientList(header("dynamicEndpoint")) + .to("direct:processSets"); - // Set the dynamic endpoint URL in the exchange header - exchange.getIn().setHeader("dynamicEndpoint", endpoint); - }) - .recipientList(header("dynamicEndpoint")); - //.to("direct:processSets"); - - from("direct:processSets") + from("direct:processSets") .process(exchange -> { // Access the individual response from the ArrayList ArrayList responseList = exchange.getIn().getBody(ArrayList.class); @@ -46,46 +45,20 @@ public class OaiSetListRoute extends RouteBuilder { // Set the converted InputStream as the new body exchange.getIn().setBody(responseStream); - }) - .split(xpath("//set")).log("${body}") + }).log("${body}") + .split(xpath("//*[local-name()='set']")).log("${body}") + // .log("${body}") .aggregate(constant(true), new ListSetAggregationStrategy()) .completionPredicate(exchangeProperty(Exchange.AGGREGATED_SIZE).isEqualTo(header("CamelSplitSize"))) - .to("direct:processAggregatedSets").log("Aggregated: ${body}"); + .to("direct:processAggregatedSets") + .log("${body}"); - /* - .aggregate(header("dynamicEndpoint"), new ListSetAggregationStrategy()) - .completionPredicate(exchangeProperty(Exchange.AGGREGATED_SIZE).isEqualTo(header("CamelSplitSize"))) - .marshal().json(JsonLibrary.Jackson) - .convertBodyTo(String.class) - .log("Aggregated Response: ${body}");*/ - } - - /*from("direct:getResponse") - .to("oaipmh:http://repositorium.sdum.uminho.pt/oai/request?verb=ListSets") + from("direct:processAggregatedSets") .process(exchange -> { - // Process the API response - String apiResponse = exchange.getIn().getBody(String.class); - - // Create a new response - String newResponse = "New Response: I made it!"; - - // Set the new response in the exchange body - exchange.getIn().setBody(newResponse); - });*/ - - -/* from(oaiEndpoint) - .convertBodyTo(String.class) // Convert response to String - .marshal().json() // Convert String to JSON - .to("direct:serveResponse") - .log("JSON Response: ${body}"); - - // Serve the JSON response - from("direct:serveResponse") - .setHeader("Content-Type", constant("application/json")) // Set the response content type - .setHeader("CamelHttpResponseCode", constant(200)) // Set the HTTP response code - .setBody(simple("${body}")) // Set the response body - .log("${body}") - .to("mock:result"); // Replace 'mock:result' with the appropriate endpoint to serve the response - }*/ + String aggregatedXml = exchange.getIn().getBody(ArrayList.class) + ""; + JSONObject jsonObject = XML.toJSONObject(aggregatedXml); + String jsonString = jsonObject.toString(); + exchange.getIn().setBody(jsonString); + }).to("seda:result"); + } } diff --git a/src/main/java/eu/dnetlib/validatorapi/utils/ListSetAggregationStrategy.java b/src/main/java/eu/dnetlib/validatorapi/utils/ListSetAggregationStrategy.java index 07613a3..01ab450 100644 --- a/src/main/java/eu/dnetlib/validatorapi/utils/ListSetAggregationStrategy.java +++ b/src/main/java/eu/dnetlib/validatorapi/utils/ListSetAggregationStrategy.java @@ -2,18 +2,6 @@ package eu.dnetlib.validatorapi.utils; import org.apache.camel.AggregationStrategy; import org.apache.camel.Exchange; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; - -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.xpath.*; -import java.io.IOException; -import java.io.StringReader; public class ListSetAggregationStrategy implements AggregationStrategy { @@ -22,76 +10,12 @@ public class ListSetAggregationStrategy implements AggregationStrategy { return newExchange; } - String oldBody = null; - try { - oldBody = exctractSets(oldExchange.getIn().getBody(String.class)); - String newBody = exctractSets(newExchange.getIn().getBody(String.class)); - oldExchange.getIn().setBody(oldBody + newBody); + String oldBody = oldExchange.getIn().getBody(String.class); + String newBody = newExchange.getIn().getBody(String.class); + oldExchange.getIn().setBody(oldBody + newBody); - } catch (ParserConfigurationException e) { - System.out.println(e); - e.printStackTrace(); - } catch (IOException e) { - System.out.println(e); - - e.printStackTrace(); - } catch (SAXException e) { - System.out.println(e); - - e.printStackTrace(); - } catch (TransformerException e) { - System.out.println(e); - - e.printStackTrace(); - } catch (XPathExpressionException e) { - System.out.println(e); - - e.printStackTrace(); - } return oldExchange; } - private String exctractSets(String xml) throws ParserConfigurationException, IOException, SAXException, TransformerException, XPathExpressionException { - // Create an XPathFactory - XPathFactory xfactory = XPathFactory.newInstance(); - - // Create an XPath object - XPath xpath = xfactory.newXPath(); - - // Compile the XPath expression to select the set elements - XPathExpression expression = xpath.compile("//set"); - - // Create an InputSource from the XML string - InputSource inputSource = new InputSource(new StringReader(xml)); - - // Parse the inputSource and create a Document object - Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(inputSource); - - // Evaluate the XPath expression on the document - NodeList setElements = (NodeList) expression.evaluate(document, XPathConstants.NODESET); - - // Process the extracted set elements - StringBuilder extractedElements = new StringBuilder(); - for (int i = 0; i < setElements.getLength(); i++) { - extractedElements.append(getElementAsString(setElements.item(i))); - } - // System.out.println(extractedElements.toString()); - return extractedElements.toString(); - } - - // Helper method to convert a Node to String - private static String getElementAsString(Node node) { - try { - javax.xml.transform.TransformerFactory tf = javax.xml.transform.TransformerFactory.newInstance(); - javax.xml.transform.Transformer transformer = tf.newTransformer(); - transformer.setOutputProperty(javax.xml.transform.OutputKeys.OMIT_XML_DECLARATION, "yes"); - java.io.StringWriter writer = new java.io.StringWriter(); - transformer.transform(new javax.xml.transform.dom.DOMSource(node), new javax.xml.transform.stream.StreamResult(writer)); - return writer.toString(); - } catch (Exception e) { - e.printStackTrace(); - return ""; - } - } } \ No newline at end of file