correct score calculated + json returned for ListSets + internal error taken into account
This commit is contained in:
parent
e61e252306
commit
97388284f2
49
pom.xml
49
pom.xml
|
@ -89,40 +89,6 @@
|
|||
<artifactId>camel-spring-boot-starter</artifactId>
|
||||
<version>${apache.camel.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.camel.springboot</groupId>
|
||||
<artifactId>camel-jackson-starter</artifactId>
|
||||
<version>${apache.camel.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.camel</groupId>
|
||||
<artifactId>camel-jackson</artifactId>
|
||||
<version>${apache.camel.version}</version> <!-- Use the appropriate version -->
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.camel</groupId>
|
||||
<artifactId>camel-jacksonxml</artifactId>
|
||||
<version>${apache.camel.version}</version>
|
||||
<!-- use the same version as your Camel core version -->
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-core</artifactId>
|
||||
<version>2.15.2</version> <!-- Use the appropriate version -->
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>2.15.2</version> <!-- Use the appropriate version -->
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||
<artifactId>jackson-dataformat-xml</artifactId>
|
||||
<version>2.15.2</version> <!-- Use the appropriate version -->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.camel.springboot</groupId>
|
||||
<artifactId>camel-servlet-starter</artifactId>
|
||||
|
@ -133,10 +99,17 @@
|
|||
<artifactId>camel-http</artifactId>
|
||||
<version>${apache.camel.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- JSON -->
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||
<artifactId>jackson-dataformat-xml</artifactId>
|
||||
<version>2.15.2</version>
|
||||
<groupId>org.json</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>20230618</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.xml.bind</groupId>
|
||||
<artifactId>jaxb-api</artifactId>
|
||||
<version>2.3.1</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
@ -145,7 +118,7 @@
|
|||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<version>1.5.22.RELEASE</version>
|
||||
<version>2.7.10</version>
|
||||
<configuration>
|
||||
<mainClass>eu.dnetlib.validatorapi.ValidatorApiApplication</mainClass>
|
||||
<executable>true</executable>
|
||||
|
|
|
@ -13,6 +13,7 @@ import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
|
|||
import eu.dnetlib.validatorapi.routes.OaiPmhRoute;
|
||||
import eu.dnetlib.validatorapi.routes.OaiSetListRoute;
|
||||
import org.apache.camel.CamelContext;
|
||||
import org.apache.camel.ConsumerTemplate;
|
||||
import org.apache.camel.ProducerTemplate;
|
||||
import org.apache.camel.builder.RouteBuilder;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
|
@ -81,7 +82,6 @@ public class ValidationController {
|
|||
log.error("Exception: No valid guidelines");
|
||||
new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided.");
|
||||
}
|
||||
System.out.println("AAAAAAAA");
|
||||
ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords);
|
||||
validationJob.guidelines = profile.name();
|
||||
validationJobRepository.save(validationJob);
|
||||
|
@ -240,8 +240,11 @@ public class ValidationController {
|
|||
|
||||
RouteBuilder listSetsBuilder = new OaiSetListRoute();
|
||||
ProducerTemplate producerTemplate = camelContext.createProducerTemplate();
|
||||
String response = producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);
|
||||
return response;
|
||||
producerTemplate.requestBodyAndHeader("direct:getResponse", null, "endpoint", "oaipmh://"+ baseURL + "?verb=ListSets", String.class);
|
||||
|
||||
ConsumerTemplate consumerTemplate = camelContext.createConsumerTemplate();
|
||||
String result = consumerTemplate.receive("seda:result").getIn().getBody(String.class);
|
||||
return result;
|
||||
|
||||
/*
|
||||
System.out.println(camelContext.getRoutes());
|
||||
|
|
|
@ -12,13 +12,16 @@ public class ValidationJob {
|
|||
|
||||
@Column(name = "base_url")
|
||||
public String baseUrl;
|
||||
|
||||
@Column(name="number_of_records")
|
||||
public int numberOfRecords;
|
||||
|
||||
@Column(name="guidelines")
|
||||
public String guidelines;
|
||||
|
||||
@Column(name="started")
|
||||
public Date startDate;
|
||||
|
||||
@Column(name="ended")
|
||||
public Date endDate;
|
||||
|
||||
|
|
|
@ -27,7 +27,8 @@ public class XmlProcessor implements Processor {
|
|||
private final ValidationResultRepository validationResultRepository;
|
||||
private final long maxNumberOfRecords;
|
||||
private int processedRecords;
|
||||
private int resultSum;
|
||||
// private int resultSum;
|
||||
private int scoreSum;
|
||||
|
||||
|
||||
public XmlProcessor(final AbstractOpenAireProfile profile, ValidationJob validationJob,
|
||||
|
@ -56,13 +57,14 @@ public class XmlProcessor implements Processor {
|
|||
Map<String, Guideline.Result> results = validationResult.results();
|
||||
|
||||
ValidationRuleResult validationRuleResult;
|
||||
for (Map.Entry entry : results.entrySet()) {
|
||||
for (Map.Entry entry : results.entrySet()) { //for each rule of the record
|
||||
final Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||
String ruleName = entry.getKey().toString();
|
||||
validationRuleResult = new ValidationRuleResult();
|
||||
constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl,
|
||||
ruleName, profile, engineResult);
|
||||
|
||||
|
||||
if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) {
|
||||
validationRuleResult.hasErrors = true;
|
||||
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR");
|
||||
|
@ -74,13 +76,14 @@ public class XmlProcessor implements Processor {
|
|||
}
|
||||
|
||||
validationResultRepository.save(validationRuleResult);
|
||||
resultSum += engineResult.score();
|
||||
scoreSum += engineResult.score();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
processedRecords++;
|
||||
validationJob.recordsTested = processedRecords;
|
||||
validationJob.score = resultSum;
|
||||
validationJob.score = scoreSum/processedRecords;
|
||||
//System.out.println("\n\n\nvalidation job: " + validationJob.recordsTested);
|
||||
|
||||
if (processedRecords > maxNumberOfRecords) {
|
||||
|
|
|
@ -28,7 +28,11 @@ public interface ValidationResultRepository extends JpaRepository<ValidationRule
|
|||
@Query(value = "SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," +
|
||||
"COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passedRecords," +
|
||||
"COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failedRecords," +
|
||||
"(SELECT CASE WHEN COUNT(DISTINCT CASE WHEN vr.status = 'FAILURE' THEN vr.ruleName END) = COUNT(DISTINCT vr.ruleName) THEN 'FAILURE' ELSE 'SUCCESS' END " +
|
||||
"(SELECT CASE" +
|
||||
" WHEN COUNT(DISTINCT CASE WHEN vr.status = 'FAILURE' THEN vr.ruleName END) = COUNT(DISTINCT vr.ruleName) THEN 'FAILURE'" +
|
||||
" WHEN COUNT(DISTINCT CASE WHEN vr.status = 'ERROR' THEN vr.ruleName END) = COUNT(DISTINCT vr.ruleName) THEN 'ERROR'" +
|
||||
" ELSE 'SUCCESS'" +
|
||||
" END " +
|
||||
"FROM eu.dnetlib.validatorapi.entities.ValidationRuleResult vr WHERE vr.ruleName = sr.ruleName)," +
|
||||
"CASE WHEN COUNT(CASE WHEN has_errors IS TRUE THEN 1 END) > 0 THEN TRUE ELSE FALSE END AS hasErrors," +
|
||||
"CASE WHEN COUNT(CASE WHEN has_warnings IS TRUE THEN 1 END) > 0 THEN TRUE ELSE FALSE END AS hasWarnings)" +
|
||||
|
|
|
@ -3,7 +3,8 @@ package eu.dnetlib.validatorapi.routes;
|
|||
import eu.dnetlib.validatorapi.utils.ListSetAggregationStrategy;
|
||||
import org.apache.camel.Exchange;
|
||||
import org.apache.camel.builder.RouteBuilder;
|
||||
import org.apache.camel.model.dataformat.JacksonXMLDataFormat;
|
||||
import org.json.JSONObject;
|
||||
import org.json.XML;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
|
@ -17,6 +18,7 @@ public class OaiSetListRoute extends RouteBuilder {
|
|||
|
||||
public String xml = new String();
|
||||
|
||||
|
||||
/*public OaiSetListRoute(final String oaiEndpoint) {
|
||||
this.oaiEndpoint = oaiEndpoint;
|
||||
}*/
|
||||
|
@ -24,21 +26,18 @@ public class OaiSetListRoute extends RouteBuilder {
|
|||
|
||||
@Override
|
||||
public void configure() throws Exception {
|
||||
JacksonXMLDataFormat xmlJsonFormat = new JacksonXMLDataFormat();
|
||||
|
||||
|
||||
from("direct:getResponse")
|
||||
.process(exchange -> {
|
||||
// Get the endpoint URL from the request parameter
|
||||
String endpoint = exchange.getIn().getHeader("endpoint", String.class);
|
||||
.process(exchange -> {
|
||||
// Get the endpoint URL from the request parameter
|
||||
String endpoint = exchange.getIn().getHeader("endpoint", String.class);
|
||||
// Set the dynamic endpoint URL in the exchange header
|
||||
exchange.getIn().setHeader("dynamicEndpoint", endpoint);
|
||||
})
|
||||
.recipientList(header("dynamicEndpoint"))
|
||||
.to("direct:processSets");
|
||||
|
||||
// Set the dynamic endpoint URL in the exchange header
|
||||
exchange.getIn().setHeader("dynamicEndpoint", endpoint);
|
||||
})
|
||||
.recipientList(header("dynamicEndpoint"));
|
||||
//.to("direct:processSets");
|
||||
|
||||
from("direct:processSets")
|
||||
from("direct:processSets")
|
||||
.process(exchange -> {
|
||||
// Access the individual response from the ArrayList
|
||||
ArrayList<?> responseList = exchange.getIn().getBody(ArrayList.class);
|
||||
|
@ -46,46 +45,20 @@ public class OaiSetListRoute extends RouteBuilder {
|
|||
|
||||
// Set the converted InputStream as the new body
|
||||
exchange.getIn().setBody(responseStream);
|
||||
})
|
||||
.split(xpath("//set")).log("${body}")
|
||||
}).log("${body}")
|
||||
.split(xpath("//*[local-name()='set']")).log("${body}")
|
||||
// .log("${body}")
|
||||
.aggregate(constant(true), new ListSetAggregationStrategy())
|
||||
.completionPredicate(exchangeProperty(Exchange.AGGREGATED_SIZE).isEqualTo(header("CamelSplitSize")))
|
||||
.to("direct:processAggregatedSets").log("Aggregated: ${body}");
|
||||
.to("direct:processAggregatedSets")
|
||||
.log("${body}");
|
||||
|
||||
/*
|
||||
.aggregate(header("dynamicEndpoint"), new ListSetAggregationStrategy())
|
||||
.completionPredicate(exchangeProperty(Exchange.AGGREGATED_SIZE).isEqualTo(header("CamelSplitSize")))
|
||||
.marshal().json(JsonLibrary.Jackson)
|
||||
.convertBodyTo(String.class)
|
||||
.log("Aggregated Response: ${body}");*/
|
||||
}
|
||||
|
||||
/*from("direct:getResponse")
|
||||
.to("oaipmh:http://repositorium.sdum.uminho.pt/oai/request?verb=ListSets")
|
||||
from("direct:processAggregatedSets")
|
||||
.process(exchange -> {
|
||||
// Process the API response
|
||||
String apiResponse = exchange.getIn().getBody(String.class);
|
||||
|
||||
// Create a new response
|
||||
String newResponse = "New Response: I made it!";
|
||||
|
||||
// Set the new response in the exchange body
|
||||
exchange.getIn().setBody(newResponse);
|
||||
});*/
|
||||
|
||||
|
||||
/* from(oaiEndpoint)
|
||||
.convertBodyTo(String.class) // Convert response to String
|
||||
.marshal().json() // Convert String to JSON
|
||||
.to("direct:serveResponse")
|
||||
.log("JSON Response: ${body}");
|
||||
|
||||
// Serve the JSON response
|
||||
from("direct:serveResponse")
|
||||
.setHeader("Content-Type", constant("application/json")) // Set the response content type
|
||||
.setHeader("CamelHttpResponseCode", constant(200)) // Set the HTTP response code
|
||||
.setBody(simple("${body}")) // Set the response body
|
||||
.log("${body}")
|
||||
.to("mock:result"); // Replace 'mock:result' with the appropriate endpoint to serve the response
|
||||
}*/
|
||||
String aggregatedXml = exchange.getIn().getBody(ArrayList.class) + "";
|
||||
JSONObject jsonObject = XML.toJSONObject(aggregatedXml);
|
||||
String jsonString = jsonObject.toString();
|
||||
exchange.getIn().setBody(jsonString);
|
||||
}).to("seda:result");
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,18 +2,6 @@ package eu.dnetlib.validatorapi.utils;
|
|||
|
||||
import org.apache.camel.AggregationStrategy;
|
||||
import org.apache.camel.Exchange;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.xpath.*;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
|
||||
public class ListSetAggregationStrategy implements AggregationStrategy {
|
||||
|
||||
|
@ -22,76 +10,12 @@ public class ListSetAggregationStrategy implements AggregationStrategy {
|
|||
return newExchange;
|
||||
}
|
||||
|
||||
String oldBody = null;
|
||||
try {
|
||||
oldBody = exctractSets(oldExchange.getIn().getBody(String.class));
|
||||
String newBody = exctractSets(newExchange.getIn().getBody(String.class));
|
||||
oldExchange.getIn().setBody(oldBody + newBody);
|
||||
String oldBody = oldExchange.getIn().getBody(String.class);
|
||||
String newBody = newExchange.getIn().getBody(String.class);
|
||||
oldExchange.getIn().setBody(oldBody + newBody);
|
||||
|
||||
} catch (ParserConfigurationException e) {
|
||||
System.out.println(e);
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
System.out.println(e);
|
||||
|
||||
e.printStackTrace();
|
||||
} catch (SAXException e) {
|
||||
System.out.println(e);
|
||||
|
||||
e.printStackTrace();
|
||||
} catch (TransformerException e) {
|
||||
System.out.println(e);
|
||||
|
||||
e.printStackTrace();
|
||||
} catch (XPathExpressionException e) {
|
||||
System.out.println(e);
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
return oldExchange;
|
||||
}
|
||||
|
||||
|
||||
private String exctractSets(String xml) throws ParserConfigurationException, IOException, SAXException, TransformerException, XPathExpressionException {
|
||||
// Create an XPathFactory
|
||||
XPathFactory xfactory = XPathFactory.newInstance();
|
||||
|
||||
// Create an XPath object
|
||||
XPath xpath = xfactory.newXPath();
|
||||
|
||||
// Compile the XPath expression to select the set elements
|
||||
XPathExpression expression = xpath.compile("//set");
|
||||
|
||||
// Create an InputSource from the XML string
|
||||
InputSource inputSource = new InputSource(new StringReader(xml));
|
||||
|
||||
// Parse the inputSource and create a Document object
|
||||
Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(inputSource);
|
||||
|
||||
// Evaluate the XPath expression on the document
|
||||
NodeList setElements = (NodeList) expression.evaluate(document, XPathConstants.NODESET);
|
||||
|
||||
// Process the extracted set elements
|
||||
StringBuilder extractedElements = new StringBuilder();
|
||||
for (int i = 0; i < setElements.getLength(); i++) {
|
||||
extractedElements.append(getElementAsString(setElements.item(i)));
|
||||
}
|
||||
// System.out.println(extractedElements.toString());
|
||||
return extractedElements.toString();
|
||||
}
|
||||
|
||||
// Helper method to convert a Node to String
|
||||
private static String getElementAsString(Node node) {
|
||||
try {
|
||||
javax.xml.transform.TransformerFactory tf = javax.xml.transform.TransformerFactory.newInstance();
|
||||
javax.xml.transform.Transformer transformer = tf.newTransformer();
|
||||
transformer.setOutputProperty(javax.xml.transform.OutputKeys.OMIT_XML_DECLARATION, "yes");
|
||||
java.io.StringWriter writer = new java.io.StringWriter();
|
||||
transformer.transform(new javax.xml.transform.dom.DOMSource(node), new javax.xml.transform.stream.StreamResult(writer));
|
||||
return writer.toString();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue