diff --git a/pom.xml b/pom.xml index 9eea48e..1411454 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,11 @@ camel-spring-boot ${apache.camel.version} + + org.apache.camel.springboot + camel-core-starter + ${apache.camel.version} + org.apache.camel.springboot camel-spring-boot-starter diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java index b63ff27..117f253 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ReportController.java @@ -2,16 +2,14 @@ package eu.dnetlib.validatorapi.controllers; import eu.dnetlib.validatorapi.entities.SummaryResult; import eu.dnetlib.validatorapi.entities.ValidationIssue; +import eu.dnetlib.validatorapi.entities.ValidationJob; import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @RestController @CrossOrigin(origins = "*") @@ -29,9 +27,13 @@ public class ReportController { this.validationIssueRepository = validationIssueRepository; } - + @RequestMapping(value={"getJobResult"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) + public ValidationJob getJobResults(@RequestParam(name = "jobId") int jobId){ + Optional validationJob = validationJobRepository.findById(jobId); + return (ValidationJob) validationJob.orElse(null); + } @RequestMapping(value = {"getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public List getJobResults(@RequestParam(name = "jobId") int jobId){ + public List getSummaryJobResults(@RequestParam(name = "jobId") int jobId){ return validationResultRepository.getSummaryResult(jobId); } diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index 0dd67a4..cda9171 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -78,10 +78,11 @@ public class ValidationController { log.error("Exception: No valid guidelines"); new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided."); } - + System.out.println("AAAAAAAA"); ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords); validationJob.guidelines = profile.name(); validationJobRepository.save(validationJob); + System.out.println("id " + validationJob.id); int record = 0; double resultSum = 0; @@ -125,6 +126,7 @@ public class ValidationController { } record++; + System.out.println(record++); } } validationJob.status = "COMPLETED"; @@ -141,7 +143,7 @@ public class ValidationController { validationJob.recordsTested = record; validationJob.score = resultSum / record; //TODO uncomment - // validationJobRepository.save(validationJob); + validationJobRepository.save(validationJob); } //xmlValidationResponse.setRules(resultRules); @@ -179,9 +181,10 @@ public class ValidationController { @RequestMapping(value = {"/realValidator"}, method = RequestMethod.GET) public void validateWithApacheCamel(@RequestParam(name = "guidelines") String guidelinesProfileName, - @RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now - @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now - @RequestParam(name="set", required = false) String set, @RequestParam(name="metadataPrefix", defaultValue = "oai_dc") String metadataPrefix) { + @RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now + @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, + @RequestParam(name="set", required = false) String set, //not in use now + @RequestParam(name="metadataPrefix", defaultValue = "oai_dc") String metadataPrefix) { AbstractOpenAireProfile profile = initializeOpenAireProfile(guidelinesProfileName); AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName); @@ -194,7 +197,10 @@ public class ValidationController { ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords); validationJob.guidelines = profile.name(); + validationJob.status = "IN_PROGRESS"; validationJobRepository.save(validationJob); + log.info("Initial validation job id "+ validationJob.id); + int record = 0; double resultSum = 0; @@ -207,8 +213,6 @@ public class ValidationController { new OaiPmhRoute("oaipmh://"+baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix , profile, validationJob, numberOfRecords, validationIssueRepository, validationResultRepository); camelContext.addRoutes(oaiPmhRouteBuilder); - //camelContext.start(); - //camelContext.stop(); validationJob.status = "COMPLETED"; } @@ -219,7 +223,7 @@ public class ValidationController { } finally { validationJob.endDate = new Date(); - System.out.println("Final validation job "+ validationJob.hashCode()); + log.info("Final validation job "+ validationJob.id); validationJob.recordsTested = record; validationJob.score = resultSum / record; validationJobRepository.save(validationJob); @@ -378,8 +382,9 @@ public class ValidationController { AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName); validationJob.guidelines = profile.name(); - System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n"); validationJobRepository.save(validationJob); + log.info("Initial validation job id "+ validationJob.id); + int record = 0; double resultSum = 0; @@ -425,19 +430,15 @@ public class ValidationController { } catch (Exception e) { log.error("Validation job stopped unexpectedly." + e.getMessage()); - System.out.println("ERROR " + e.getMessage()); validationJob.status = "STOPPED"; } finally { validationJob.endDate = new Date(); - System.out.println("Final validation job "+ validationJob.hashCode()); + log.info("Final validation job "+ validationJob.id); validationJob.recordsTested = record; validationJob.score = resultSum / record; validationJobRepository.save(validationJob); } - - //xmlValidationResponse.setRules(resultRules); - //xmlValidationResponse.setFairRules(fairRules); } diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java index a128bd9..ae753bb 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java @@ -25,7 +25,7 @@ public class ValidationJob { @Column(name = "records_tested") public int recordsTested; @Column(name="duration") - public String status; //stopped, completed, in progres + public String status; //stopped, completed, in progress @Column(name="score") public double score; diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java new file mode 100644 index 0000000..1c7b143 --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/processors/StopRouteProcessor.java @@ -0,0 +1,4 @@ +package eu.dnetlib.validatorapi.processors; + +public class StopRouteProcessor { +} diff --git a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java index d0babd3..bf5d9a8 100644 --- a/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java +++ b/src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java @@ -23,35 +23,33 @@ import java.util.Map; public class XmlProcessor implements Processor { private final AbstractOpenAireProfile profile; private final ValidationJob validationJob; - private long processedRecords; - private final ValidationIssueRepository validationIssueRepository; private final ValidationResultRepository validationResultRepository; + private final long maxNumberOfRecords; + private long processedRecords; + public XmlProcessor(final AbstractOpenAireProfile profile, final ValidationJob validationJob, final ValidationIssueRepository validationIssueRepository, - final ValidationResultRepository validationResultRepository){ + final ValidationResultRepository validationResultRepository, + final long maxNumberOfRecords){ super(); this.profile = profile; this.validationJob = validationJob; this.validationIssueRepository = validationIssueRepository; this.validationResultRepository = validationResultRepository; + this.maxNumberOfRecords = maxNumberOfRecords; } @Override public void process(Exchange exchange) throws Exception { String recordXml = exchange.getIn().getBody(String.class); - final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); final DocumentBuilder documentBuilder = dbf.newDocumentBuilder(); Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml))); String recordUrl = extractRecordUrl(doc, "identifier"); - processedRecords++; - System.out.println("processed " + processedRecords); - exchange.setProperty("totalRecords", processedRecords); - if (profile != null) { XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that? System.out.println("DC IDENTIFIER: " + recordUrl); @@ -66,100 +64,31 @@ public class XmlProcessor implements Processor { constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl, ruleName, profile, engineResult); validationResultRepository.save(validationRuleResult); - saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult); - - // System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n"); - } - - } - } - - private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) { - for (String error:engineResult.errors()) { - /*System.out.println("11111");*/ - ValidationIssue validationIssue = new ValidationIssue(); - validationIssue.validationJobId = validationJobId; - validationIssue.ruleName = ruleName; - validationIssue.recordUrl = recordUrl; - validationIssue.issueType = "ERROR"; - validationIssue.issueText = error; - validationIssueRepository.save(validationIssue); -/* - System.out.println(validationIssue); -*/ - } - - for (String warning: engineResult.warnings()){ -/* - System.out.println("22222"); -*/ - ValidationIssue validationIssue = new ValidationIssue(); - validationIssue.validationJobId = validationJobId; - validationIssue.ruleName = ruleName; - validationIssue.recordUrl = recordUrl; - validationIssue.issueType = "WARNING"; - validationIssue.issueText = warning; - validationIssueRepository.save(validationIssue); -/* - System.out.println(validationIssue); -*/ - - } - - } - - /*public void process2(Exchange exchange) throws Exception { - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - String recordXml = exchange.getIn().getBody(String.class); - - // System.out.println("\n\n\n\n\n\n\n\n\n\n\n\n"); - System.out.println(recordXml); - // System.out.println("\n\n\n\n\n\n\n\n\n\n\n\n"); - - Document doc = db.parse(new InputSource(new StringReader(recordXml))); - - - if (profile != null) { - XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that? - String recordUrl = extractRecordUrl(doc, "identifier"); - System.out.println("DC IDENTIFIER: " + recordUrl); - - Map results = validationResult.results(); - for (Map.Entry entry : results.entrySet()) { - Guideline.Result engineResult = (Guideline.Result) entry.getValue(); - // System.out.println("engine result > " + engineResult); - - String ruleName = entry.getKey().toString(); - // System.out.println("rulename: " + recordUrl); - - ValidationRuleResult validationRuleResult = new ValidationRuleResult(); - // System.out.println("validationjob.id " + validationJob.id); - // System.out.println("recordUrl " + recordUrl); - // System.out.println("rulename " + ruleName); - // System.out.println("profile " + profile); - // System.out.println("engineResult " + engineResult); - - - - constructValidationRuleResult(validationRuleResult, validationJob.id, recordUrl, - ruleName, profile, engineResult); - - // System.out.println("xaxaxa"); - // System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n"); - // resultSum += engineResult.score(); - - // elements = nodeListToList(recordNodes); // assign the Elements to the variable - - - + saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.errors(), "ERROR"); + saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult.warnings(), "WARNING"); } } processedRecords++; - System.out.println("processed " + processedRecords); exchange.setProperty("totalRecords", processedRecords); - }*/ + + if (processedRecords > maxNumberOfRecords) { + exchange.getIn().setHeader("MyHeader", "stop"); + } + } + + private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, + Iterable issues, String issueType) { + for (String issue:issues) { + ValidationIssue validationIssue = new ValidationIssue(); + validationIssue.validationJobId = validationJobId; + validationIssue.ruleName = ruleName; + validationIssue.recordUrl = recordUrl; + validationIssue.issueType = issueType; + validationIssue.issueText = issue; + validationIssueRepository.save(validationIssue); + } + } //TODO consider throwing exception - UTIL class? private String extractRecordUrl(final Document document, final String xmlField) { diff --git a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationJobRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationJobRepository.java index 38893f1..c115a30 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationJobRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationJobRepository.java @@ -5,6 +5,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; @Repository -public interface ValidationJobRepository extends JpaRepository { +public interface ValidationJobRepository extends JpaRepository { } diff --git a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java index 7ff4a97..efae172 100644 --- a/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java +++ b/src/main/java/eu/dnetlib/validatorapi/routes/OaiPmhRoute.java @@ -9,6 +9,9 @@ import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.apache.camel.builder.RouteBuilder; +import java.util.Date; +import java.util.concurrent.CountDownLatch; + public class OaiPmhRoute extends RouteBuilder { private String oaiEndpoint; @@ -18,6 +21,8 @@ public class OaiPmhRoute extends RouteBuilder { private ValidationJob validationJob; private final ValidationIssueRepository validationIssueRepository; private final ValidationResultRepository validationResultRepository; + CountDownLatch latch; + public OaiPmhRoute(String oaiEndpoint, AbstractOpenAireProfile profile, ValidationJob validationJob, long maxNumberOfRecords, final ValidationIssueRepository validationIssueRepository, @@ -28,21 +33,100 @@ public class OaiPmhRoute extends RouteBuilder { this.maxNumberOfRecords = maxNumberOfRecords; this.validationIssueRepository = validationIssueRepository; this.validationResultRepository = validationResultRepository; + this.latch = latch; } @Override public void configure() throws Exception { + String date = new Date().toString(); - from(oaiEndpoint) - .setProperty("totalRecords", constant(0)) - .loopDoWhile().simple("${exchangeProperty.totalRecords} < 10") + from(oaiEndpoint).routeId(date) .split(xpath("//*[local-name()='record']")) + .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository, maxNumberOfRecords)) + .choice().when(header("MyHeader").isEqualTo("stop")) + .to("controlbus:route?routeId="+date+"&action=stop") + .endChoice(); + + /*from("timer://myTimer?fixedRate=true&period=60000") + .routeId("OAIProcessingRoute") + .setHeader("MyHeader", constant("stop")) + .choice() + .when(header("MyHeader").isEqualTo("stop")) + .toD("direct:oaiEndpoint") + .otherwise() + .stop() + .end(); + + from("timer://myTimer?fixedRate=true&period=600000") + .to(oaiEndpoint) + .log("hello") + .split(xpath("//*[local-name()='record']")) + .log("${body}") + .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository)) + .log("????"); + +/* from("direct:processXML") + //.split(xpath("//*[local-name()='record']")) + .log("${exchangeProperty.totalRecords}") + .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository)) + .log("${exchangeProperty.totalRecords}"); + + + + String date = new Date().toString(); +/* + + from(oaiEndpoint) + .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository)) + .to("mock:start") + .process(new Processor() { + @Override + public void process(Exchange exchange) throws Exception { + // stop Camel by signalling to the latch + latch.countDown(); + } + }).to("mock:done"); + +*/ +/* + from(oaiEndpoint) + .routeId(date) + .setHeader("MyHeader", constant("start")) + .choice() + .when(header("MyHeader").isEqualTo("stop")) + .log("Myheader " +"${header.myHeader}") + + .to("controlbus:route?routeId="+date+"&action=stop") + .stop() + .endChoice() + .otherwise() + .process(exchange -> exchange.getIn().setHeader("MyHeader", "stop")) + .log("Trying to make the thread stop! " +"${header.myHeader}") + .endChoice(); + */ + + /* from(oaiEndpoint) + .setProperty("totalRecords", constant(0)) + .loopDoWhile(simple("${exchangeProperty.totalRecords} < 5")) + .log("${exchangeProperty.totalRecords}") + //.split(xpath("//*[local-name()='record']")) + //.log("\n\n\n----------------\n\n\n\n\n\n${body}\n\n\n----------------\n\n\n\n\n\n") + .to("direct:ListRecords") + .end(); + + from("direct:ListRecords") + .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository)); + /* from(oaiEndpoint) + .setProperty("totalRecords", constant(0)) + .loopDoWhile(simple("${exchangeProperty.totalRecords} < 10")) + .split(xpath("//*[local-name()='record']")).streaming() .log("${exchangeProperty.totalRecords}") // .log("\n\n\n----------------\n\n\n\n\n\n${body}\n\n\n----------------\n\n\n\n\n\n") .process(new XmlProcessor(profile, validationJob, validationIssueRepository, validationResultRepository)) - .end().to("direct:end"); - + .log("${exchangeProperty.totalRecords}") + .end().to("direct:end"); +*/ /* from("oaipmh://http://repositorium.sdum.uminho.pt/oai/request?verb=ListRecords&metadataPrefix=oai_dc") .setProperty("totalRecords", constant(0)) .loopDoWhile().simple("${exchangeProperty.totalRecords} < 10") @@ -60,7 +144,7 @@ public class OaiPmhRoute extends RouteBuilder { .process(new XmlProcessor(profile, validationJob)) .end() .to("direct:end"); -**/ +/***/ //from(oaiEndpoint).process(new XmlProcessor(profile,validationJob)); // from("direct:oaiRequest").process(new DummyXMLProcessor()); @@ -122,8 +206,7 @@ public class OaiPmhRoute extends RouteBuilder { class RecordCountProcessor implements Processor { @Override public void process(Exchange exchange) throws Exception { - System.out.println("RecordCountProcessor"); - + System.out.println("RecordCountProcessor reads"); int processedRecords = exchange.getProperty("processedRecords", 0, Integer.class); exchange.setProperty("totalRecords", processedRecords);