Compare commits
2 Commits
f031a79718
...
50cb62af1e
Author | SHA1 | Date |
---|---|---|
Katerina | 50cb62af1e | |
Katerina | 8929ff38ab |
|
@ -226,8 +226,6 @@ public class ValidationController {
|
|||
oaiPmhRouteBuilder =
|
||||
new SimpleOaiPmhRoute(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
|
||||
} else {
|
||||
System.out.println("THERE");
|
||||
|
||||
oaiPmhRouteBuilder =
|
||||
new FairOaiPmhRoute2(oaiEndpoint, profile, validationJob, numberOfRecords, uuid.toString());
|
||||
}
|
||||
|
@ -394,7 +392,8 @@ public class ValidationController {
|
|||
|
||||
private String initializeMetadataPrefix(String guidelinesProfileName) {
|
||||
|
||||
if (guidelinesProfileName.equals("OpenAIRE Guidelines for Data Archives Profile v2")) {
|
||||
if (guidelinesProfileName.equals("OpenAIRE Guidelines for Data Archives Profile v2") ||
|
||||
guidelinesProfileName.equals("OpenAIRE FAIR Guidelines for Data Repositories Profile")) {
|
||||
return "oai_datacite";
|
||||
|
||||
} else if (guidelinesProfileName.equals("OpenAIRE Guidelines for Literature Repositories Profile v3")) {
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
package eu.dnetlib.validatorapi.controllers;
|
||||
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@CrossOrigin(origins = "*")
|
||||
@RequestMapping("/jobs")
|
||||
public class ValidationJobController {
|
||||
|
||||
@Autowired
|
||||
ValidationJobRepository validationJobRepository;
|
||||
|
||||
@RequestMapping(value = {"/latest"}, method = RequestMethod.GET)
|
||||
public List<ValidationJob> getLatestJobs(@RequestParam(name="limit", defaultValue = "10") int limit) {
|
||||
return validationJobRepository.getLatestValidationJobs(PageRequest.of(0, limit));
|
||||
}
|
||||
}
|
|
@ -0,0 +1,37 @@
|
|||
package eu.dnetlib.validatorapi.processors;
|
||||
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import org.apache.camel.Exchange;
|
||||
import org.apache.camel.Processor;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class ExceptionProcessor implements Processor {
|
||||
|
||||
ValidationJob validationJob;
|
||||
|
||||
public ExceptionProcessor(ValidationJob validationJob) {
|
||||
this.validationJob = validationJob;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Exchange exchange) {
|
||||
Throwable caused = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Throwable.class);
|
||||
if (caused.getClass() != null) {
|
||||
if (caused.getClass().getName() != null) {
|
||||
validationJob.exceptionClass = caused.getClass().getName();
|
||||
}
|
||||
}
|
||||
|
||||
if (caused.getCause() != null) {
|
||||
if (caused.getCause().getMessage() != null) {
|
||||
validationJob.exceptionMessage = caused.getCause().toString();
|
||||
}
|
||||
}
|
||||
|
||||
validationJob.progress = "STOPPED";
|
||||
validationJob.endDate = new Date();
|
||||
|
||||
exchange.getIn().setBody(validationJob, ValidationJob.class);
|
||||
}
|
||||
}
|
|
@ -128,15 +128,24 @@ public class XmlProcessor implements Processor {
|
|||
validationRuleResult.validationJobId = validationJobId;
|
||||
validationRuleResult.guidelines = guidelines;
|
||||
validationRuleResult.ruleName = ruleName;
|
||||
validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight();
|
||||
validationRuleResult.requirement_level = profile.guideline(ruleName).getRequirementLevel().name();
|
||||
System.out.println(validationRuleResult.requirement_level);
|
||||
validationRuleResult.description = profile.guideline(ruleName).getDescription();
|
||||
System.out.println(validationRuleResult.description);
|
||||
validationRuleResult.fair_principles = profile.guideline(ruleName).getFairPrinciples();
|
||||
System.out.println(validationRuleResult.fair_principles);
|
||||
validationRuleResult.link = profile.guideline(ruleName).getLink();
|
||||
System.out.println(validationRuleResult.link);
|
||||
|
||||
if (profile.guideline(ruleName) != null) {
|
||||
validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight();
|
||||
|
||||
if (profile.guideline(ruleName).getRequirementLevel() != null
|
||||
&& profile.guideline(ruleName).getRequirementLevel().name()!=null)
|
||||
validationRuleResult.requirement_level = profile.guideline(ruleName).getRequirementLevel().name();
|
||||
|
||||
if (profile.guideline(ruleName).getDescription() != null)
|
||||
validationRuleResult.description = profile.guideline(ruleName).getDescription();
|
||||
|
||||
if (profile.guideline(ruleName).getFairPrinciples() != null)
|
||||
validationRuleResult.fair_principles = profile.guideline(ruleName).getFairPrinciples();
|
||||
|
||||
if (profile.guideline(ruleName).getLink() != null)
|
||||
validationRuleResult.link = profile.guideline(ruleName).getLink();
|
||||
}
|
||||
|
||||
validationRuleResult.recordUrl = recordUrl;
|
||||
validationRuleResult.score = engineResult.score();
|
||||
validationRuleResult.status = engineResult.status().toString();
|
||||
|
|
|
@ -1,10 +1,15 @@
|
|||
package eu.dnetlib.validatorapi.repositories;
|
||||
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Repository
|
||||
public interface ValidationJobRepository extends JpaRepository<ValidationJob, Integer> {
|
||||
|
||||
@Query(value ="SELECT vj FROM ValidationJob vj ORDER BY vj.startDate DESC")
|
||||
List<ValidationJob> getLatestValidationJobs(Pageable pageable);
|
||||
}
|
||||
|
|
|
@ -6,10 +6,13 @@ import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
|||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
|
||||
import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
|
||||
import eu.dnetlib.validatorapi.processors.ExceptionProcessor;
|
||||
import eu.dnetlib.validatorapi.processors.XmlProcessor;
|
||||
import org.apache.camel.Exchange;
|
||||
import org.apache.camel.Processor;
|
||||
import org.apache.camel.TypeConversionException;
|
||||
import org.apache.camel.builder.RouteBuilder;
|
||||
import org.apache.http.client.ClientProtocolException;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.UUID;
|
||||
|
@ -37,9 +40,26 @@ public class FairOaiPmhRoute2 extends RouteBuilder {
|
|||
|
||||
@Override
|
||||
public void configure() throws Exception {
|
||||
|
||||
String date = new Date().toString();
|
||||
|
||||
onException(TypeConversionException.class)
|
||||
.process(new ExceptionProcessor(validationJob))
|
||||
.to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true").log("\n\n\n\nHEREEEEEE")
|
||||
.to("controlbus:route?routeId="+routeid+"&action=stop")
|
||||
.to("controlbus:route?routeId="+routeid2+"&action=stop")
|
||||
.maximumRedeliveries(0)
|
||||
.handled(true)
|
||||
.end();
|
||||
|
||||
onException(ClientProtocolException.class)
|
||||
.process(new ExceptionProcessor(validationJob))
|
||||
.to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")
|
||||
.to("controlbus:route?routeId="+routeid+"&action=stop")
|
||||
.to("controlbus:route?routeId="+routeid2+"&action=stop")
|
||||
.maximumRedeliveries(0)
|
||||
.handled(true)
|
||||
.end();
|
||||
|
||||
from(oaiEndpoint)
|
||||
.multicast().parallelProcessing()
|
||||
.to("direct:guidelinesProcessor")
|
||||
|
|
|
@ -1,2 +1,185 @@
|
|||
package eu.dnetlib.validatorapi.routes;public class OAI_PMH_RouteBuilder {
|
||||
package eu.dnetlib.validatorapi.routes;
|
||||
|
||||
import eu.dnetlib.validator2.validation.guideline.openaire.LiteratureGuidelinesV4Profile;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
|
||||
import eu.dnetlib.validatorapi.processors.XmlProcessor;
|
||||
import org.apache.camel.builder.RouteBuilder;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.transform.OutputKeys;
|
||||
import javax.xml.transform.Transformer;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.transform.TransformerFactory;
|
||||
import javax.xml.transform.dom.DOMSource;
|
||||
import javax.xml.transform.stream.StreamResult;
|
||||
import java.io.StringReader;
|
||||
import java.io.StringWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
public class OAI_PMH_RouteBuilder extends RouteBuilder {
|
||||
/*
|
||||
private final String oaiEndpoint = "oaipmh://http://repositorium.sdum.uminho.pt/oai/request";
|
||||
private AbstractOpenAireProfile profile;
|
||||
private long maxNumberOfRecords = 0;
|
||||
private ValidationJob validationJob;
|
||||
|
||||
private final ValidationJobRepository validationJobRepository;
|
||||
private final ValidationIssueRepository validationIssueRepository;
|
||||
private final ValidationResultRepository validationResultRepository;
|
||||
|
||||
public OAI_PMH_RouteBuilder(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob,
|
||||
long maxNumberOfRecords, final ValidationJobRepository validationJobRepository,
|
||||
final ValidationIssueRepository validationIssueRepository,
|
||||
final ValidationResultRepository validationResultRepository) {
|
||||
this.validationJob = validationJob;
|
||||
this.profile = profile;
|
||||
this.maxNumberOfRecords = maxNumberOfRecords;
|
||||
this.validationJobRepository = validationJobRepository;
|
||||
this.validationIssueRepository = validationIssueRepository;
|
||||
this.validationResultRepository = validationResultRepository;
|
||||
}*/
|
||||
|
||||
@Override
|
||||
public void configure() throws Exception {
|
||||
|
||||
/* String id1 = new Date().getTime()+"";
|
||||
String id2 = new Date().getTime()+1+"";
|
||||
String date = new Date().toString();
|
||||
*/
|
||||
from("direct:startProcessing")
|
||||
.toD("${body}") // Use dynamic endpoint from the message body (the endpoint set by OaiPmhRoute)
|
||||
.routeId("Process_OAI_Response")
|
||||
.split(xpath("//*[local-name()='record']"))
|
||||
.process(new XmlProcessor(new LiteratureGuidelinesV4Profile(), new ValidationJob(), 1))
|
||||
.choice()
|
||||
.when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
|
||||
.split(simple("${body[results]}"))
|
||||
.to("jpa:" + ValidationRuleResult.class.getName() + "?usePersist=true")
|
||||
.stop() // Stop the route when this condition is true
|
||||
.endChoice()
|
||||
.end()
|
||||
.choice()
|
||||
.when(simple("${body[issues]} && ${header.MyHeader} != 'stop'"))
|
||||
.split(simple("${body[issues]}"))
|
||||
.to("jpa:" + ValidationIssue.class.getName() + "?usePersist=true")
|
||||
.stop() // Stop the route when this condition is true
|
||||
.endChoice()
|
||||
.end()
|
||||
.choice()
|
||||
.when(header("MyHeader").isEqualTo("stop"))
|
||||
.to("direct:saveToDatabase")
|
||||
.to("controlbus:route?routeId=Process_OAI_Response&action=stop")
|
||||
.endChoice()
|
||||
.end();
|
||||
|
||||
|
||||
|
||||
|
||||
/*from("direct:getRecords")
|
||||
.toD("${header.oaiEndpoint}")
|
||||
.routeId(date)
|
||||
.split(xpath("//*[local-name()='record']"))
|
||||
.log("${body}")
|
||||
.process(new XmlProcessor(profile, validationJob, maxNumberOfRecords))
|
||||
.choice()
|
||||
.when(simple("${body[issues]} && ${header.MyHeader} != 'stop'"))
|
||||
.log("HERE!!!")
|
||||
.split(simple("${body[issues]}"))
|
||||
.to("jpa:"+ ValidationRuleResult.class.getName()+ "?usePersist=true")
|
||||
.endChoice()
|
||||
.end()
|
||||
.choice()
|
||||
.when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
|
||||
.log("THERE")
|
||||
.split(simple("${body[results]}"))
|
||||
.to("jpa:" + ValidationIssue.class.getName() + "?usePersist=true")
|
||||
.endChoice()
|
||||
.end()
|
||||
.choice()
|
||||
.when(header("MyHeader").isEqualTo("stop"))
|
||||
.to("direct:saveToDatabase")
|
||||
.to("controlbus:route?routeId="+date+"&action=stop")
|
||||
.endChoice()
|
||||
.end();
|
||||
|
||||
|
||||
from("direct:saveToDatabase")
|
||||
.process(new DataBaseProcessor())
|
||||
.to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true");*/
|
||||
|
||||
/*
|
||||
from(oaiEndpoint)
|
||||
.multicast().parallelProcessing()
|
||||
.to("direct:guidelinesProcessor")
|
||||
.to("direct:fairProcessor")
|
||||
.end();
|
||||
|
||||
from("direct:guidelinesProcessor")
|
||||
.routeId(id1)
|
||||
.split(xpath("//*[local-name()='record']"))
|
||||
//TODO .process(new XmlProcessor(profile, validationJob, maxNumberOfRecords))
|
||||
//.log("PROCESSOR 1111 \n\n\n ${body}")
|
||||
.choice().when(header("MyHeader").isEqualTo("stop"))
|
||||
//.to("direct:saveToDatabase")
|
||||
.to("controlbus:route?routeId="+id1+"&action=stop")
|
||||
.endChoice();
|
||||
|
||||
from("direct:fairProcessor")
|
||||
.routeId(id2)
|
||||
.split(xpath("//*[local-name()='record']"))
|
||||
//.log("PROCESSOR 222 \n\n\n ${body}")
|
||||
//TODO .process(new XmlProcessor(new FAIR_Data_GuidelinesProfile(), validationJob, maxNumberOfRecords))
|
||||
.choice().when(header("MyHeader").isEqualTo("stop"))
|
||||
//.to("direct:saveToDatabase")
|
||||
.to("controlbus:route?routeId="+id2+"&action=stop")
|
||||
.endChoice();
|
||||
|
||||
|
||||
from("direct:saveToDatabase") //TODO FIX THIS with JPA
|
||||
.process(new DataBaseProcessor());
|
||||
|
||||
|
||||
*/
|
||||
}
|
||||
|
||||
private List<String> extractSets(String xml) {
|
||||
List<String> setXmls = new ArrayList<>();
|
||||
try {
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder builder = factory.newDocumentBuilder();
|
||||
InputSource inputSource = new InputSource(new StringReader(xml));
|
||||
Document document = builder.parse(inputSource);
|
||||
|
||||
NodeList setElements = document.getElementsByTagName("set");
|
||||
for (int i = 0; i < setElements.getLength(); i++) {
|
||||
Node setElement = setElements.item(i);
|
||||
String setXml = nodeToString(setElement);
|
||||
setXmls.add(setXml);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// Handle any exceptions that occur during XML parsing or processing
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
|
||||
return setXmls;
|
||||
}
|
||||
|
||||
private String nodeToString(Node node) throws TransformerException {
|
||||
StringWriter sw = new StringWriter();
|
||||
TransformerFactory tf = TransformerFactory.newInstance();
|
||||
Transformer transformer = tf.newTransformer();
|
||||
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
|
||||
transformer.transform(new DOMSource(node), new StreamResult(sw));
|
||||
return sw.toString();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package eu.dnetlib.validatorapi.routes;
|
||||
|
||||
import org.apache.camel.builder.RouteBuilder;
|
||||
import org.apache.camel.impl.engine.DefaultShutdownStrategy;
|
||||
import org.json.JSONObject;
|
||||
import org.json.XML;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
@ -28,6 +29,12 @@ public class OaiSetListRoute extends RouteBuilder {
|
|||
@Override
|
||||
public void configure() throws Exception {
|
||||
|
||||
// Access the DefaultShutdownStrategy
|
||||
DefaultShutdownStrategy shutdownStrategy = (DefaultShutdownStrategy) getContext().getShutdownStrategy();
|
||||
|
||||
// Set the shutdown timeout in milliseconds (e.g., 10 seconds)
|
||||
shutdownStrategy.setTimeout(1);
|
||||
|
||||
from("direct:getResponse")
|
||||
.process(exchange -> {
|
||||
String endpoint = exchange.getIn().getHeader("endpoint", String.class);
|
||||
|
|
|
@ -5,10 +5,13 @@ import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
|||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
|
||||
import eu.dnetlib.validatorapi.processors.DataBaseProcessor;
|
||||
import eu.dnetlib.validatorapi.processors.ExceptionProcessor;
|
||||
import eu.dnetlib.validatorapi.processors.XmlProcessor;
|
||||
import org.apache.camel.Exchange;
|
||||
import org.apache.camel.Processor;
|
||||
import org.apache.camel.TypeConversionException;
|
||||
import org.apache.camel.builder.RouteBuilder;
|
||||
import org.apache.http.client.ClientProtocolException;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
|
@ -37,9 +40,32 @@ public class SimpleOaiPmhRoute extends RouteBuilder {
|
|||
|
||||
String date = new Date().toString();
|
||||
|
||||
onException(TypeConversionException.class)
|
||||
.process(new ExceptionProcessor(validationJob))
|
||||
.to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true").log("\n\n\n\nHEREEEEEE")
|
||||
.to("controlbus:route?routeId="+routeid+"&action=stop")
|
||||
.maximumRedeliveries(0)
|
||||
.handled(true)
|
||||
.end();
|
||||
|
||||
onException(ClientProtocolException.class)
|
||||
.process(new ExceptionProcessor(validationJob))
|
||||
.to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true")
|
||||
.to("controlbus:route?routeId="+routeid+"&action=stop")
|
||||
.maximumRedeliveries(0)
|
||||
.handled(true)
|
||||
.end();
|
||||
|
||||
/*
|
||||
.process(new TypeConversionExceptionProcessor(validationJob)).log("\n\n\n\n HERE 2")
|
||||
.process(new DataBaseProcessor())//.maximumRedeliveries(0)
|
||||
.to("jpa:" + ValidationJob.class.getName() + "?useExecuteUpdate=true").log("\n\n\n\nHEREEEEEE")
|
||||
.stop();
|
||||
*/
|
||||
from(oaiEndpoint)
|
||||
.routeId(routeid)
|
||||
.split(xpath("//*[local-name()='record']"))
|
||||
.doTry()
|
||||
.process(new XmlProcessor(profile, validationJob, maxNumberOfRecords))
|
||||
.choice()
|
||||
.when(simple("${body[results]} && ${header.MyHeader} != 'stop'"))
|
||||
|
|
Loading…
Reference in New Issue