MaDgIK
/
uoa-validator-api
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrating the 871b2ec6e5 commit of validator-engine2. You need to install it locally, therefore I have added it as 1.9.9-local version in my pom.xml

This commit is contained in:
Katerina 2024-04-03 09:52:04 +03:00
parent d42d8fc09b
commit 1a99c041d3
7 changed files with 107 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -34,7 +34,7 @@
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>uoa-validator-engine2</artifactId>
<version>0.9.0</version>
<version>1.9.9-local</version>
</dependency>
<!-- spring boot -->

38
src/main/java/eu/dnetlib/validatorapi/RecordAggregationStrategy.java Normal file
View File

@ -0,0 +1,38 @@
package eu.dnetlib.validatorapi;
import org.apache.camel.AggregationStrategy;
import org.apache.camel.Exchange;
import java.util.ArrayList;
import java.util.List;
public class RecordAggregationStrategy implements AggregationStrategy {
private static final int MAX_RECORDS = 2;
@Override
public Exchange aggregate(Exchange oldExchange, Exchange newExchange) {
if (oldExchange == null) {
// First record, create a new ArrayList to store records
newExchange.getIn().setHeader("recordList", new ArrayList<>());
return newExchange;
}
// Get the list of records from the oldExchange
List<String> recordList = oldExchange.getIn().getHeader("recordList", List.class);
// Add the new record to the list
recordList.add(newExchange.getIn().getBody(String.class));
if (recordList.size() >= MAX_RECORDS) {
// Set the aggregated records as the message body
oldExchange.getIn().setBody(recordList);
// Remove the property to signal the route to stop
oldExchange.removeProperty(Exchange.ROUTE_STOP);
} else {
// Store the updated list back in the header for the next iteration
oldExchange.getIn().setHeader("recordList", recordList);
}
return oldExchange;
}
}

36
src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java
View File

@ -119,14 +119,14 @@ public class ValidationController {
if (profile != null) {
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
Map<String, Guideline.Result> results = validationResult.results();
Map<String, Guideline.Result> results = validationResult.getResults();
for (Map.Entry entry : results.entrySet()) {
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
String recordUrl = extractRecordUrl(doc, "identifier");
String ruleName = entry.getKey().toString();
ValidationRuleResult validationRuleResult = constructValidationRuleResult(validationJob.id, recordUrl,
ruleName, profile, engineResult);
resultSum += engineResult.score();
resultSum += engineResult.getScore();
validationResultRepository.save(validationRuleResult);
//saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult);
@ -309,9 +309,9 @@ public class ValidationController {
validationRuleResult.ruleWeight = profile.guideline(validationRuleResult.ruleName).getWeight();
validationRuleResult.recordUrl = extractRecordUrl(document, "dc:identifier");
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
validationRuleResult.score = engineResult.score();
validationRuleResult.status = engineResult.status().toString();
validationRuleResult.internalError = engineResult.internalError();
validationRuleResult.score = engineResult.getScore();
validationRuleResult.status = engineResult.getStatus().toString();
validationRuleResult.internalError = engineResult.getInternalError();
}
@ -325,9 +325,9 @@ public class ValidationController {
validationRuleResult.ruleWeight = profile.guideline(validationRuleResult.ruleName).getWeight();
validationRuleResult.recordUrl = extractRecordUrl(document, "dc:identifier");
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
validationRuleResult.score = engineResult.score();
validationRuleResult.status = engineResult.status().toString();
validationRuleResult.internalError = engineResult.internalError();
validationRuleResult.score = engineResult.getScore();
validationRuleResult.status = engineResult.getStatus().toString();
validationRuleResult.internalError = engineResult.getInternalError();
return validationRuleResult;
}
@ -373,9 +373,9 @@ public class ValidationController {
validationRuleResult.ruleName = ruleName;
validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight();
validationRuleResult.recordUrl = recordUrl;
validationRuleResult.score = engineResult.score();
validationRuleResult.status = engineResult.status().toString();
validationRuleResult.internalError = engineResult.internalError();
validationRuleResult.score = engineResult.getScore();
validationRuleResult.status = engineResult.getStatus().toString();
validationRuleResult.internalError = engineResult.getInternalError();
return validationRuleResult;
}
@ -432,10 +432,8 @@ public class ValidationController {
return new FAIR_Data_GuidelinesProfile();
} else if (guidelinesProfileName.equals("OpenAIRE Guidelines for Literature Repositories Profile v4")) {
System.out.println("\n\n\nRETURNING FAIR LITERATURE\n\n\n");
System.out.println("\n\n\n" + new FAIR_Literature_GuidelinesV4Profile().name() + "\n\n\n");
return new FAIR_Literature_GuidelinesV4Profile();
return new FAIR_Literature_GuidelinesV4Profile();
}
return null;
@ -496,7 +494,7 @@ public class ValidationController {
//what id is that?
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc);
Map<String, Guideline.Result> results = validationResult.results();
Map<String, Guideline.Result> results = validationResult.getResults();
for (Map.Entry entry : results.entrySet()) {
ValidationRuleResult validationRuleResult = new ValidationRuleResult();
@ -504,14 +502,14 @@ public class ValidationController {
validationRuleResult.ruleName = entry.getKey().toString();
validationRuleResult.recordUrl = "localhost://records/record["+record+"]"; // silly id
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
validationRuleResult.score = engineResult.score();
validationRuleResult.status = engineResult.status().toString();
validationRuleResult.internalError = engineResult.internalError();
validationRuleResult.score = engineResult.getScore();
validationRuleResult.status = engineResult.getStatus().toString();
validationRuleResult.internalError = engineResult.getInternalError();
System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
validationResultRepository.save(validationRuleResult);
validationRuleResults.add(validationRuleResult);
resultSum += engineResult.score();
resultSum += engineResult.getScore();
}

32
src/main/java/eu/dnetlib/validatorapi/controllers/ValidatorController.java
View File

@ -71,8 +71,8 @@ public class ValidatorController {
result = profile.validate("id", doc);
Date date = new Date();
scorePerDoc.put("input-" + date.toString(), result.score());
Map<String, Guideline.Result> results = result.results();
scorePerDoc.put("input-" + date.toString(), result.getScore());
Map<String, Guideline.Result> results = result.getResults();
log.debug("\n\nPrinting results...");
for (Map.Entry entry : results.entrySet()) {
log.debug(entry.getKey() + " = " + entry.getValue());
@ -81,11 +81,11 @@ public class ValidatorController {
ruleInfo.setName(entry.getKey().toString());
Guideline.Result res = (Guideline.Result) entry.getValue();
ruleInfo.setErrors((List<String>) res.errors());
ruleInfo.setWarnings((List<String>) res.warnings());
ruleInfo.setInternalError(res.internalError());
ruleInfo.setScore(res.score());
ruleInfo.setStatus(res.status());
ruleInfo.setErrors((List<String>) res.getErrors());
ruleInfo.setWarnings((List<String>) res.getWarnings());
ruleInfo.setInternalError(res.getInternalError());
ruleInfo.setScore(res.getScore());
ruleInfo.setStatus(res.getStatus());
resultRules.add(ruleInfo);
}
@ -107,8 +107,8 @@ public class ValidatorController {
result = fairProfile.validate("id", doc);
Date date = new Date();
scorePerDoc.put("input-"+date.toString(), result.score());
Map<String, Guideline.Result> results = result.results();
scorePerDoc.put("input-"+date.toString(), result.getScore());
Map<String, Guideline.Result> results = result.getResults();
log.debug("\n\nPrinting FAIR results...");
for (Map.Entry entry : results.entrySet()) {
// if (!entry.getValue().toString().contains("eu.dnetlib.validator2")) {
@ -118,11 +118,11 @@ public class ValidatorController {
ruleInfo.setName(entry.getKey().toString());
Guideline.Result res = (Guideline.Result) entry.getValue();
ruleInfo.setErrors((List<String>) res.errors());
ruleInfo.setWarnings((List<String>) res.warnings());
ruleInfo.setInternalError(res.internalError());
ruleInfo.setScore(res.score());
ruleInfo.setStatus(res.status());
ruleInfo.setErrors((List<String>) res.getErrors());
ruleInfo.setWarnings((List<String>) res.getWarnings());
ruleInfo.setInternalError(res.getInternalError());
ruleInfo.setScore(res.getScore());
ruleInfo.setStatus(res.getStatus());
fairRules.add(ruleInfo);
// }
@ -182,8 +182,8 @@ public class ValidatorController {
Document doc = db.parse(filePath);
result = profile.validate("id", doc);
Date date = new Date();
scorePerDoc.put("input-"+date.toString(), result.score());
Map<String, Guideline.Result> results = result.results();
scorePerDoc.put("input-"+date.toString(), result.getScore());
Map<String, Guideline.Result> results = result.getResults();
log.debug("\n\nPrinting results...");
for (Map.Entry entry : results.entrySet()) {
log.debug(entry.getKey() + " = " + entry.getValue());

12
src/main/java/eu/dnetlib/validatorapi/exceptions/CustomRejectedExecutionHandler.java Normal file
View File

@ -0,0 +1,12 @@
package eu.dnetlib.validatorapi.exceptions;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
public class CustomRejectedExecutionHandler implements RejectedExecutionHandler {
//@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
throw new RejectedExecutionException("Task rejected: " + r.toString());
}
}

29
src/main/java/eu/dnetlib/validatorapi/processors/XmlProcessor.java
View File

@ -48,13 +48,13 @@ public class XmlProcessor implements Processor {
final DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
Document doc = documentBuilder.parse(new InputSource(new StringReader(recordXml)));
String recordUrl = extractRecordUrl(doc, "identifier");
System.out.println("Record url: " + recordUrl);
List<ValidationRuleResult> validationRuleResults = new ArrayList<>();
List<ValidationIssue> validationIssues = new ArrayList<>();
System.out.println("\n\n\nProcess with guideline " + profile.name());
if (profile != null) {
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
Map<String, Guideline.Result> results = validationResult.results();
Map<String, Guideline.Result> results = validationResult.getResults();
ValidationRuleResult validationRuleResult;
for (Map.Entry entry : results.entrySet()) { //for each rule of the record
@ -68,22 +68,22 @@ public class XmlProcessor implements Processor {
this.status = "FAILURE";
}
if (engineResult.errors()!= null && engineResult.errors().iterator().hasNext()) {
if (engineResult.getErrors()!= null && engineResult.getErrors().iterator().hasNext()) {
validationRuleResult.hasErrors = true;
for (String error:engineResult.errors()) {
for (String error:engineResult.getErrors()) {
validationIssues.add(new ValidationIssue(validationJob.id, ruleName, recordUrl, error,"ERROR"));
}
}
if (engineResult.warnings()!= null && engineResult.warnings().iterator().hasNext()) {
if (engineResult.getWarnings()!= null && engineResult.getWarnings().iterator().hasNext()) {
validationRuleResult.hasWarnings = true;
for (String warning:engineResult.warnings()) {
for (String warning:engineResult.getWarnings()) {
validationIssues.add(new ValidationIssue(validationJob.id, ruleName, recordUrl, warning, "WARNING"));
}
}
validationRuleResults.add(validationRuleResult);
scoreSum += engineResult.score();
scoreSum += engineResult.getScore();
}
}
@ -98,25 +98,18 @@ public class XmlProcessor implements Processor {
// System.out.println("Validation Issues: " + validationIssues);
exchange.getIn().setHeader("MyHeader", "SKATA");
System.out.println("processed records: " + processedRecords);
if (processedRecords > maxNumberOfRecords) {
exchange.getIn().setHeader("MyHeader", "stop");
exchange.getIn().setHeader("validationId", validationJob.id);
exchange.getIn().setHeader("baseUrl", validationJob.baseUrl);
exchange.getIn().setHeader("set", validationJob.set);
exchange.getIn().setHeader("startDate", validationJob.startDate);
System.out.println("\n\n\n" + validationJob.id +": set guidelines "+ validationJob.guidelines + "\n\n\n");
exchange.getIn().setHeader("guidelines", validationJob.guidelines);
exchange.getIn().setHeader("status", status);
exchange.getIn().setHeader("recordsTested", processedRecords-1);
exchange.getIn().setHeader("maxNumberOfRecords", maxNumberOfRecords);
exchange.getIn().setHeader("score", scoreSum/processedRecords);
System.out.println("\n\n\n\n\n PLACE STOP HEADER \n\n\n\n\n\n");
//exchange.setRouteStop(true);
}
}
@ -157,8 +150,8 @@ public class XmlProcessor implements Processor {
}
validationRuleResult.recordUrl = recordUrl;
validationRuleResult.score = engineResult.score();
validationRuleResult.status = engineResult.status().toString();
validationRuleResult.internalError = engineResult.internalError();
validationRuleResult.score = engineResult.getScore();
validationRuleResult.status = engineResult.getStatus().toString();
validationRuleResult.internalError = engineResult.getInternalError();
}
}

13
src/main/resources/application.properties
View File

@ -6,4 +6,15 @@ spring.datasource.username=dnet
spring.datasource.password=pass
#drop n create table again, good for testing, comment this in production or set as validate
spring.jpa.hibernate.ddl-auto=update
spring.jpa.hibernate.ddl-auto=update
#springdoc.packagesToScan=eu.dnetlib.validatorapi.controllers
#eu.dnetlib.validatorapi.controllers=/v1
#spring.mvc.pathmatch.matching-strategy=ant-path-matcher
#logging.level.org.apache.camel=error
logging.level.eu.dnetlib.validator2.validation.guideline=OFF
#camel.springboot.tracing=true
logging.level.org.hibernate.SQL=info
#logging.level.org.hibernate.type.descriptor.sql=trace