MaDgIK
/
uoa-validator-engine2
13
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of code-repo.d4science.org:MaDgIK/validator-engine 

# Conflicts:
#	pom.xml
#	src/test/java/eu/dnetlib/validator2/engine/Test_v4.java
This commit is contained in:
pispis 2023-07-10 14:57:41 +03:00
parent 5eb062dffd fd99f536d3
commit 92c5c97ab4
19 changed files with 301 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
pom.xml
View File

@ -13,44 +13,8 @@
<!-- <timestamp>${maven.build.timestamp}</timestamp>-->
<!-- <maven.build.timestamp.format>E MMM dd HH:mm:ss z yyyy</maven.build.timestamp.format>-->
<!-- </properties>-->
<dependencies>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.20.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.20.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.spockframework</groupId>
<artifactId>spock-core</artifactId>
<version>1.3-groovy-2.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.15</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
@ -66,11 +30,61 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
<version>3.3.2</version>
<version>3.4.0</version>
<configuration>
<failOnMissingWebXml>false</failOnMissingWebXml>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.13.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>32.1.1-jre</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.spockframework</groupId>
<artifactId>spock-core</artifactId>
<version>1.3-groovy-2.5</version>
<scope>test</scope>
</dependency>
<!-- logback versions 1.4.X require Java-11 -->
<!-- logback versions 1.3.X require Java-8, but id this project is added as Dependency in Spring Boot, then Spring Boot throws an error, since it does not yet support logback 1.3.x -->
<!-- https://mvnrepository.com/artifact/ch.qos.logback/logback-core -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.2.12</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.36</version>
</dependency>
<!-- https://mvnrepository.com/artifact/ch.qos.logback/logback-classic -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.12</version>
</dependency>
</dependencies>
</project>

10
src/main/java/eu/dnetlib/validator2/engine/Helper.java
View File

@ -4,6 +4,7 @@ import eu.dnetlib.validator2.validation.guideline.Builders;
import eu.dnetlib.validator2.validation.guideline.Builders.ElementSpecBuilder;
import eu.dnetlib.validator2.validation.guideline.Cardinality;
import eu.dnetlib.validator2.validation.guideline.RequirementLevel;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -24,6 +25,9 @@ import java.util.zip.ZipInputStream;
public class Helper {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(Helper.class);
private static final String EMPTY = "";
public static boolean isEmpty(String s) {
@ -116,15 +120,15 @@ public class Helper {
int len = nodes.getLength();
for (int i = 0; i < len; i++) {
Node node = nodes.item(i);
// System.out.println("Getting node: " + node);
//logger.debug("Getting node: " + node);
T t = nodeReader.apply(node);
// System.out.println("Read node value: " + t);
//logger.debug("Read node value: " + t);
if (predicate.test(t)) {
filtered.add(node);
}
}
}
// System.out.println(filtered.size() + " nodes matched the predicate");
//logger.debug(filtered.size() + " nodes matched the predicate");
return new ListOfNodes(filtered);
}

23
src/main/java/eu/dnetlib/validator2/engine/Reporter.java
View File

@ -1,7 +1,12 @@
package eu.dnetlib.validator2.engine;
import org.slf4j.LoggerFactory;
public class Reporter<T, R extends Rule<T>> {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(Reporter.class);
private final RuleDiagnostics<T, R> diagnostics;
public Reporter(RuleDiagnostics<T, R> diagnostics) {
@ -11,30 +16,24 @@ public class Reporter<T, R extends Rule<T>> {
public void reportSuccessFor(R rule, T t) {
try {
diagnostics.success(rule, t);
}
catch(Throwable throwable) {
System.err.println("Failed to report success of applying " + rule + " to value: " + t);
throwable.printStackTrace();
} catch (Throwable throwable) {
logger.error("Failed to report success of applying " + rule + " to value: " + t, throwable);
}
}
public void reportFailureFor(R rule, T t) {
try {
diagnostics.failure(rule, t);
}
catch(Throwable throwable) {
System.err.println("Failed to report failure of applying " + rule + " to value: " + t);
throwable.printStackTrace();
} catch (Throwable throwable) {
logger.error("Failed to report failure of applying " + rule + " to value: " + t, throwable);
}
}
public void reportErrorFor(R rule, T t, Throwable throwable) {
try {
diagnostics.error(rule, t, throwable);
}
catch(Throwable throwable1) {
System.err.println("Failed to report error of applying " + rule + " to value: " + t);
throwable1.printStackTrace();
} catch (Throwable throwable1) {
logger.error("Failed to report error of applying " + rule + " to value: " + t, throwable);
}
}
}

5
src/main/java/eu/dnetlib/validator2/engine/builtins/SimpleRule.java
View File

@ -3,14 +3,13 @@ package eu.dnetlib.validator2.engine.builtins;
import eu.dnetlib.validator2.engine.Rule;
import eu.dnetlib.validator2.engine.RuleContext;
import eu.dnetlib.validator2.engine.RuleEvaluationException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.function.Predicate;
public class SimpleRule<T, C extends RuleContext> implements Rule<T> {
private static final Logger logger = LogManager.getLogger();
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(SimpleRule.class);
private final C context;
private final Predicate<T> predicate;

5
src/main/java/eu/dnetlib/validator2/engine/builtins/XMLRule.java
View File

@ -3,8 +3,7 @@ package eu.dnetlib.validator2.engine.builtins;
import eu.dnetlib.validator2.engine.Rule;
import eu.dnetlib.validator2.engine.RuleEvaluationException;
import eu.dnetlib.validator2.engine.contexts.XMLContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
@ -16,7 +15,7 @@ import java.util.function.Predicate;
*/
public class XMLRule<C extends XMLContext> implements Rule<Document> {
private static final Logger logger = LogManager.getLogger();
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(XMLRule.class);
private final C context;
public final Predicate<NodeList> predicate;

4
src/main/java/eu/dnetlib/validator2/engine/builtins/XMLValidURLRule.java
View File

@ -10,9 +10,7 @@ import java.util.function.Predicate;
public class XMLValidURLRule extends XMLRule<XMLContext> {
protected XMLValidURLRule(XMLContext context) {
super(context, (NodeList nodes) -> {
return context.getNodeListActionProperty().test(nodes, (Predicate<String>) Helper.URLResolver::resolve);
});
super(context, (NodeList nodes) -> context.getNodeListActionProperty().test(nodes, (Predicate<String>) Helper.URLResolver::resolve));
}
public static Builder builder() {

5
src/main/java/eu/dnetlib/validator2/engine/builtins/XPathExpressionHelper.java
View File

@ -4,8 +4,7 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import eu.dnetlib.validator2.engine.RuleEvaluationException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -17,7 +16,7 @@ import java.util.concurrent.ExecutionException;
public class XPathExpressionHelper {
private static final Logger logger = LogManager.getLogger();
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(XPathExpressionHelper.class);
// XPath expression compilation
private static final XPath XPATH = XPathFactory.newInstance().newXPath();

7
src/main/java/eu/dnetlib/validator2/engine/contexts/CardinalityContext.java
View File

@ -2,11 +2,16 @@ package eu.dnetlib.validator2.engine.contexts;
import eu.dnetlib.validator2.engine.Helper;
import eu.dnetlib.validator2.engine.RuleContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.function.Predicate;
public interface CardinalityContext extends RuleContext {
Logger logger = LoggerFactory.getLogger(CardinalityContext.class);
String GREATER_THAN_PROPERTY_NAME = "gt";
String LESS_THAN_PROPERTY_NAME = "lt";
String UPPER_BOUND_PROPERTY_NAME = LESS_THAN_PROPERTY_NAME;
@ -20,7 +25,7 @@ public interface CardinalityContext extends RuleContext {
BooleanRuleProperty getIsInclusiveProperty();
default Predicate<Integer> cardinalityPredicate() {
// System.out.println("Evaluating cardinality " + getIdProperty().getValue());
logger.debug("Evaluating cardinality " + getIdProperty().getValue());
long min = getLowerBoundProperty().getLongValue();
long max = getUpperBoundProperty().getLongValue();

11
src/main/java/eu/dnetlib/validator2/validation/guideline/ElementSpecCompiler.java
View File

@ -9,8 +9,8 @@ import eu.dnetlib.validator2.engine.builtins.StandardXMLContext;
import eu.dnetlib.validator2.engine.builtins.XMLRule;
import eu.dnetlib.validator2.engine.builtins.XPathExpressionHelper;
import eu.dnetlib.validator2.engine.contexts.XMLContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -25,7 +25,7 @@ import java.util.stream.Stream;
class ElementSpecCompiler {
private static final Logger logger = LogManager.getLogger();
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ElementSpecCompiler.class);
private static final String[] EMPTY = new String[]{};
@ -288,6 +288,9 @@ class ElementSpecCompiler {
private static class ElementStruct {
private static final Logger logger = LoggerFactory.getLogger(ElementStruct.class);
Supplier<GuidelineEvaluation> runtimeInfo;
final ElementSpec spec;
final String[] parentElementNames;
@ -334,7 +337,7 @@ class ElementSpecCompiler {
if (spec.position().xpath != null) {
xpath = "(" + xpath + ")[" + spec.position().xpath + "]";
}
// System.out.println(xpath);
logger.debug(xpath);
return xpathWithText(xpath, withText);
}

15
src/main/java/eu/dnetlib/validator2/validation/guideline/GuidelineEvaluation.java
View File

@ -2,6 +2,8 @@ package eu.dnetlib.validator2.validation.guideline;
import eu.dnetlib.validator2.engine.*;
import eu.dnetlib.validator2.engine.builtins.StandardRuleDiagnostics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
@ -12,6 +14,9 @@ import java.util.Map;
class GuidelineEvaluation {
private static final Logger logger = LoggerFactory.getLogger(GuidelineEvaluation.class);
// TODO: Report all rule diagnostics to System out/err (we should remove this or allow clients to disable it)
private static final RuleDiagnostics<Document, Rule<Document>> OUT = Helper.Diagnostics.systemOut();
private static final RuleDiagnostics<Document, Rule<Document>> ERR = Helper.Diagnostics.systemErr();
@ -40,7 +45,7 @@ class GuidelineEvaluation {
rules.add(result.rootNodeRule);
rules.addAll(result.nodeRules);
// System.out.println("Evaluating " + rules);
logger.debug("Evaluating " + rules);
for (SyntheticRule<Document> rule: rules) {
@ -58,7 +63,7 @@ class GuidelineEvaluation {
// Report the non-applicability of a rule as a warning
// The check for both status and non-applicable requirement level is redundant
// (non-applicable rules always succeed), yet it is more clear hopefully.
// System.out.println("Warn for non-applicable: " + rule);
logger.warn("Non-applicable: " + rule);
warnings.add(synthesizeFailureMessage(rule));
}
else if (status == Status.FAILURE) {
@ -70,7 +75,7 @@ class GuidelineEvaluation {
// This is the root rule failing!
// Fail fast here, too (don't waste resources to evaluate other rules).
// We will "enable" it, if it is requested.
// System.out.println("Fail fast for root failure: " + rule);
logger.error("Fail fast for root failure: " + rule);
errors.add(synthesizeFailureMessage(rule));
return StandardResult.forFailure(warnings, errors);
}
@ -79,7 +84,7 @@ class GuidelineEvaluation {
// (a) is non-mandatory or
// (b) it was successful.
// Thus, here we need only to warn and not to err, allowing the evaluation loop to proceed.
// System.out.println("Warn for mandatory failure: " + rule);
logger.warn("Mandatory failure: " + rule);
warnings.add(synthesizeFailureMessage(rule));
}
}
@ -87,7 +92,7 @@ class GuidelineEvaluation {
// This is a warning: a non-mandatory rule has failed.
// Note that MA rules are treated as non-mandatory.
// We let the evaluation loop proceed.
// System.out.println("Warn for optional/recommended failure: " + rule);
logger.warn("Optional/Recommended failure: " + rule);
warnings.add(synthesizeFailureMessage(rule));
}
}

5
src/main/java/eu/dnetlib/validator2/validation/guideline/SyntheticGuideline.java
View File

@ -4,13 +4,12 @@ import eu.dnetlib.validator2.engine.Helper;
import eu.dnetlib.validator2.engine.Rule;
import eu.dnetlib.validator2.engine.RuleContext;
import eu.dnetlib.validator2.engine.RuleEvaluationException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
public final class SyntheticGuideline extends AbstractGuideline<Document> {
private static final Logger logger = LogManager.getLogger();
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(SyntheticGuideline.class);
private CompilationResult compilationResult;
private static final ThreadLocal<GuidelineEvaluation> evaluation = new ThreadLocal<>();

7
src/main/java/eu/dnetlib/validator2/validation/guideline/openaire/AbstractOpenAireProfile.java
View File

@ -10,6 +10,8 @@ import eu.dnetlib.validator2.validation.StandardValidationResult;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.ElementSpec;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -19,6 +21,9 @@ import java.util.Map;
public abstract class AbstractOpenAireProfile implements XMLApplicationProfile {
private static final Logger logger = LoggerFactory.getLogger(AbstractOpenAireProfile.class);
private final String name;
public AbstractOpenAireProfile(String name) {
@ -42,7 +47,7 @@ public abstract class AbstractOpenAireProfile implements XMLApplicationProfile {
results.put(guideline.getName(), result);
score += (result.status() == Status.SUCCESS ? result.score() : 0);
// System.out.println("Score after " + guideline.getName() + " = " + score);
logger.debug("Score after " + guideline.getName() + " = " + score);
}
double percentScore = (score / maxScore) * 100;

197
src/main/java/eu/dnetlib/validator2/validation/guideline/openaire/FAIR_Data_GuidelinesProfile.java
View File

@ -5,6 +5,8 @@ package eu.dnetlib.validator2.validation.guideline.openaire;
import eu.dnetlib.validator2.engine.Status;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import java.util.*;
@ -15,6 +17,8 @@ import static eu.dnetlib.validator2.validation.guideline.Cardinality.ONE_TO_N;
public final class FAIR_Data_GuidelinesProfile extends AbstractOpenAireProfile {
private static final Logger logger = LoggerFactory.getLogger(FAIR_Data_GuidelinesProfile.class);
private static final String[] TITLE_TYPES = {
"AlternativeTitle", "Subtitle", "TranslatedTitle", "Other"
};
@ -291,82 +295,86 @@ public final class FAIR_Data_GuidelinesProfile extends AbstractOpenAireProfile {
//// TODO this goes to FAIRProfile
//class MetadataCompleteness extends AbstractGuideline<Document> {
//
// public MetadataCompleteness() {
// super("MetadataCompleteness", 40);
// }
//
// @Override
// public Result validate(String id, Document t) {
// DataArchiveGuidelinesV2Profile profile = new DataArchiveGuidelinesV2Profile();
//
// // <></>ODO: iterate over results and build one Guideline.Result
// try {
//// System.out.println("Processing MetadataCompleteness...");
// XMLApplicationProfile.ValidationResult res = profile.validate(id, t);
// Map<String, Result> results = res.results();
// int MaxScoreMetadataCompleteness = (int) ((res.score()*getWeight())/100);
//
//// System.out.println("Max score DataValidator(%): " + res.score());
//// System.out.println("Weight FAIRG: " + getWeight());
//// System.out.println("Max score MetadataCompleteness: " + MaxScoreMetadataCompleteness);
//// System.out.println("\n\n\n\n");
//
//// for (Map.Entry entry : results.entrySet()) {
//// System.out.println(entry.getKey() + " = " + entry.getValue());
//// }
//// System.out.println(score);
// return getResult(MaxScoreMetadataCompleteness);
//
//// System.out.println(tempp.status() + " - " + tempp.score());
//// String printout = results.entrySet().stream().
//// map(entry -> entry.getValue() + ": " + entry.getKey()).collect(Collectors.joining("\n"));
//// System.out.println(printout);
//
//// System.out.println("\n\n\n\n");
// } catch (Exception e) {
// System.out.println(e.getMessage());
// System.out.println(e);
// e.printStackTrace();
// }
//
// return null;
// }
//
// private static Result getResult(int score) {
// String aa;
// aa = (score > 0) ? "SUCCESS" : "FAILURE";
// return new Result() {
// @Override
// public int score() {
// return score;
// }
//
// @Override
// public Status status() {
//// return null;
// return Status.valueOf(aa);
// }
//
// @Override
// public Iterable<String> warnings() { return null; }
//
// @Override
// public Iterable<String> errors() {
// return null;
// }
//
// @Override
// public String internalError() {
// return null;
// }
// };
// }
//}
/*class MetadataCompleteness extends AbstractGuideline<Document> {
private static final Logger logger = LoggerFactory.getLogger(MetadataCompleteness.class);
public MetadataCompleteness() {
super("MetadataCompleteness", 40);
}
@Override
public Result validate(String id, Document t) {
DataArchiveGuidelinesV2Profile profile = new DataArchiveGuidelinesV2Profile();
// <></>ODO: iterate over results and build one Guideline.Result
try {
// System.out.println("Processing MetadataCompleteness...");
XMLApplicationProfile.ValidationResult res = profile.validate(id, t);
Map<String, Result> results = res.results();
int MaxScoreMetadataCompleteness = (int) ((res.score()*getWeight())/100);
logger.debug("Max score DataValidator(%): " + res.score());
logger.debug("Weight FAIRG: " + getWeight());
logger.debug("Max score MetadataCompleteness: " + MaxScoreMetadataCompleteness);
logger.debug("\n\n\n\n");
if ( logger.isDebugEnabled() ) {
for ( Map.Entry entry : results.entrySet() ) {
logger.debug(entry.getKey() + " = " + entry.getValue());
}
}
logger.debug(results.entrySet().stream().
map(entry -> entry.getValue() + ": " + entry.getKey()).collect(Collectors.joining("\n")));
return getResult(MaxScoreMetadataCompleteness);
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
e.printStackTrace();
}
return null;
}
private static Result getResult(int score) {
String aa;
aa = (score > 0) ? "SUCCESS" : "FAILURE";
return new Result() {
@Override
public int score() {
return score;
}
@Override
public Status status() {
// return null;
return Status.valueOf(aa);
}
@Override
public Iterable<String> warnings() { return null; }
@Override
public Iterable<String> errors() {
return null;
}
@Override
public String internalError() {
return null;
}
};
}
}*/
class F2_01M_SPEC extends AbstractGuideline<Document> {
private static final Logger logger = LoggerFactory.getLogger(F2_01M_SPEC.class);
public F2_01M_SPEC() {
super("Rich metadata is provided to allow discovery", 2*6);
}
@ -382,9 +390,9 @@ class F2_01M_SPEC extends AbstractGuideline<Document> {
// Get actual score and not (%) to incorporate to FAIR score
final int MaxScoreF2_01M_SPEC = (int) ((res_F.score()*getWeight())/100);
// System.out.println("Max score DataValidator(%): " + res_F.score());
// System.out.println("Weight FAIRG: " + getWeight());
// System.out.println("Max score F2_01M_SPEC: " + MaxScoreF2_01M_SPEC);
logger.debug("Max score DataValidator(%): " + res_F.score());
logger.debug("Weight FAIRG: " + getWeight());
logger.debug("Max score F2_01M_SPEC: " + MaxScoreF2_01M_SPEC);
List<String> warnings2 = new ArrayList<>();
List<String> errors2 = new ArrayList<>();
@ -400,8 +408,8 @@ class F2_01M_SPEC extends AbstractGuideline<Document> {
if (entry.getValue().toString().contains("SUCCESS")) {
score += 2;
}
// System.out.println(res_F.results().get(entry.getKey()).warnings().getClass());
// System.out.println(warnings2.getClass());
logger.debug(String.valueOf(res_F.results().get(entry.getKey()).warnings().getClass()));
logger.debug(String.valueOf(warnings2.getClass()));
}
@ -410,9 +418,7 @@ class F2_01M_SPEC extends AbstractGuideline<Document> {
return new StandardResult(ress.score(), ress.status(), (List<String>) ress.warnings(), (List<String>) ress.errors(), ress.internalError());
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
e.printStackTrace();
logger.error("", e);
return null;
}
}
@ -451,6 +457,9 @@ class F2_01M_SPEC extends AbstractGuideline<Document> {
class F3_01M_SPEC extends AbstractGuideline<Document> {
private static final Logger logger = LoggerFactory.getLogger(F3_01M_SPEC.class);
public F3_01M_SPEC() { super("Metadata includes the identifier for the data", 2*2); }
@Override
@ -488,9 +497,7 @@ class F3_01M_SPEC extends AbstractGuideline<Document> {
return new StandardResult(ress.score(), ress.status(), (List<String>) ress.warnings(), (List<String>) ress.errors(), ress.internalError());
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
e.printStackTrace();
logger.error("", e);
return null;
}
}
@ -527,6 +534,8 @@ class F3_01M_SPEC extends AbstractGuideline<Document> {
class I2_01M_SPEC extends AbstractGuideline<Document> {
private static final Logger logger = LoggerFactory.getLogger(I2_01M_SPEC.class);
public I2_01M_SPEC() { super("Metadata uses FAIR-compliant vocabularies", 5*2); }
@Override
@ -535,7 +544,7 @@ class I2_01M_SPEC extends AbstractGuideline<Document> {
// TODO: iterate over results and build one Guideline.Result
try {
// System.out.println("\nMetadata uses FAIR-compliant vocabularies");
logger.debug("\nMetadata uses FAIR-compliant vocabularies");
XMLApplicationProfile.ValidationResult res_F = profile.validate(id, t);
Map<String, Result> results = res_F.results();
// int MaxScoreI2_01M_SPEC = (int) ((res_F.score()*getWeight())/100);
@ -564,9 +573,7 @@ class I2_01M_SPEC extends AbstractGuideline<Document> {
return new StandardResult(ress.score(), ress.status(), (List<String>) ress.warnings(), (List<String>) ress.errors(), ress.internalError());
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
e.printStackTrace();
logger.error("", e);
}
return null;
}
@ -604,6 +611,9 @@ class I2_01M_SPEC extends AbstractGuideline<Document> {
class R1_01M_SPEC extends AbstractGuideline<Document> {
private static final Logger logger = LoggerFactory.getLogger(R1_01M_SPEC.class);
public R1_01M_SPEC() { super("Plurality of accurate and relevant attributes are provided to allow reuse", 3*4); }
@Override
@ -637,9 +647,7 @@ class R1_01M_SPEC extends AbstractGuideline<Document> {
return new StandardResult(ress.score(), ress.status(), (List<String>) ress.warnings(), (List<String>) ress.errors(), ress.internalError());
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
e.printStackTrace();
logger.error("", e);
}
return null;
}
@ -677,6 +685,9 @@ class R1_01M_SPEC extends AbstractGuideline<Document> {
class R1_2_01M_SPEC extends AbstractGuideline<Document> {
private static final Logger logger = LoggerFactory.getLogger(R1_2_01M_SPEC.class);
public R1_2_01M_SPEC() { super("Metadata includes provenance information according to a cross-community language", 2*5); }
@Override
@ -711,9 +722,7 @@ class R1_2_01M_SPEC extends AbstractGuideline<Document> {
return new StandardResult(ress.score(), ress.status(), (List<String>) ress.warnings(), (List<String>) ress.errors(), ress.internalError());
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
e.printStackTrace();
logger.error("", e);
}
return null;
}

5
src/main/java/eu/dnetlib/validator2/validation/utils/ISOLangCodes.java
View File

@ -4,6 +4,7 @@ package eu.dnetlib.validator2.validation.utils;
import org.apache.commons.io.IOUtils;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
@ -20,10 +21,10 @@ public class ISOLangCodes {
private static void loadLangCodes() {
try (InputStream in = MediaTypes.class.getClassLoader().getResourceAsStream("iso639XLangCodes.csv")) {
//TODO:Remove the regex
List<String> l = Arrays.asList(IOUtils.toString(in, "UTF-8").split("\\s*,\\s*"));
List<String> l = Arrays.asList(IOUtils.toString(in, StandardCharsets.UTF_8).split("\\s*,\\s*"));
langs = new HashSet<>(l);
}
catch (Exception e) {}
catch (Exception e) {} // We may get an NPE.
}

16
src/test/java/eu/dnetlib/validator2/engine/Test.java
View File

@ -4,6 +4,7 @@ import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.openaire.LiteratureGuidelinesV3Profile;
import groovy.xml.DOMBuilder;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import java.io.FileReader;
@ -13,6 +14,9 @@ import java.util.stream.Collectors;
public class Test {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(Test.class);
private static final String[] FILES = new String[] {
// "src/test/resources/openaireguidelinesV3/dia.library.tuc.gr/Record_21811.xml",
// "src/test/resources/openaireguidelinesV3/cris.vtt.fi/01.xml",
@ -23,28 +27,26 @@ public class Test {
public static void main(String[] args) {
// String xmlFile = args[0];
LiteratureGuidelinesV3Profile profile = new LiteratureGuidelinesV3Profile();
System.out.println("Max score: " + profile.maxScore());
logger.info("Max score: " + profile.maxScore());
Map<String, Double> scorePerDoc = new LinkedHashMap<>();
for (String file: FILES) {
try {
System.out.println("Processing \"" + file + "\"");
logger.info("Processing \"" + file + "\"");
Document doc = DOMBuilder.parse(new FileReader(file), false, true, true);
XMLApplicationProfile.ValidationResult result = profile.validate(file, doc);
scorePerDoc.put(file, result.score());
Map<String, Guideline.Result> results = result.results();
for (Map.Entry entry : results.entrySet()) {
System.out.println(entry.getKey() + " = " + entry.getValue());
logger.info(entry.getKey() + " = " + entry.getValue());
}
}
catch(Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
e.printStackTrace();
logger.error("", e);
}
}
String printout = scorePerDoc.entrySet().stream().
map(entry -> entry.getValue() + ": " + entry.getKey()).collect(Collectors.joining("\n"));
System.out.println(printout);
logger.info(printout);
}
}

37
src/test/java/eu/dnetlib/validator2/engine/Test_FAIR.java
View File

@ -1,24 +1,23 @@
package eu.dnetlib.validator2.engine;
//import com.google.gson.Gson;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.StandardResult;
import eu.dnetlib.validator2.validation.guideline.openaire.FAIR_Data_GuidelinesProfile;
import groovy.xml.DOMBuilder;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import java.io.FileReader;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.OptionalDouble;
import java.util.StringJoiner;
import java.util.function.ToDoubleFunction;
import java.util.stream.Collectors;
import java.util.stream.DoubleStream;
public class Test_FAIR {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(Test_FAIR.class);
private static final String[] FILES = new String[]{
"src/test/resources/openaireguidelinesV3/dia.library.tuc.gr/Record_21811.xml",
"src/test/resources/openaireguidelinesV3/cris.vtt.fi/03.xml",
@ -32,27 +31,25 @@ public class Test_FAIR {
public static void main(String[] args) {
// String xmlFile = args[0];
FAIR_Data_GuidelinesProfile profile = new FAIR_Data_GuidelinesProfile();
System.out.println("Max score: " + profile.maxScore());
logger.info("Max score: " + profile.maxScore());
Map<String, Double> scorePerDoc = new LinkedHashMap<>();
int i = 1;
for (String file : FILES) {
try {
System.out.println("Processing \"" + file + "\"");
logger.info("Processing \"" + file + "\"");
Document doc = DOMBuilder.parse(new FileReader(file), false, true, true);
XMLApplicationProfile.ValidationResult result = profile.validate(file, doc);
scorePerDoc.put(file, result.score());
Map<String, Guideline.Result> results = result.results();
// for (Map.Entry entry : results.entrySet()) {
// System.out.println(entry.getKey() + " = " + entry.getValue());
// }
if ( logger.isDebugEnabled() ) {
for ( Map.Entry entry : results.entrySet() ) {
logger.debug(entry.getKey() + " = " + entry.getValue());
}
}
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
e.printStackTrace();
logger.error("", e);
}
// System.out.println("\n\n\n\n");
}
// Individual scores
@ -60,13 +57,11 @@ public class Test_FAIR {
map(entry -> entry.getValue() + ": " + entry.getKey()).collect(Collectors.joining("\n"));
// Average Score
double FinalScore = scorePerDoc.entrySet().stream().mapToDouble(entry -> entry.getValue()).average().getAsDouble();
System.out.println(printout);
System.out.println("\nValidation Score: "+FinalScore);
// TotalScore.forEach(System.out::println);
OptionalDouble optionalFinalScore = scorePerDoc.values().stream().mapToDouble(aDouble -> aDouble).average();
double finalScore = (optionalFinalScore.isPresent() ? optionalFinalScore.getAsDouble() : -1 );
logger.info(printout);
logger.info("\nValidation Score: " + finalScore);
}
}

36
src/test/java/eu/dnetlib/validator2/engine/Test_v4.java
View File

@ -2,62 +2,66 @@ package eu.dnetlib.validator2.engine;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.openaire.AbstractOpenAireProfile;
import eu.dnetlib.validator2.validation.guideline.openaire.LiteratureGuidelinesV4Profile;
import groovy.xml.DOMBuilder;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import java.io.FileReader;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.OptionalDouble;
import java.util.stream.Collectors;
public class Test_v4 {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(Test_v4.class);
private static final String[] FILES = new String[] {
"src/test/resources/openaireguidelinesV4/v4_literature_all_invalid_guidelines_record.xml",
"src/test/resources/openaireguidelinesV4/v4_literature_all_guidelines_record.xml",
// "src/test/resources/openaireguidelinesV4/v4_literature_all_guidelines_record.xml",
// "src/test/resources/openaireguidelinesV4/oai_mediarep_org_doc_2534.xml",
// "src/test/resources/openaireguidelinesV4/01_gv4.xml"
};
public static void main(String[] args) {
// String xmlFile = args[0];
LiteratureGuidelinesV4Profile profile = new LiteratureGuidelinesV4Profile();
System.out.println("Max score: " + profile.maxScore());
AbstractOpenAireProfile profile = new LiteratureGuidelinesV4Profile();
logger.info("Max score: " + profile.maxScore());
Map<String, Double> scorePerDoc = new LinkedHashMap<>();
for (String file: FILES) {
try {
System.out.println("Processing \"" + file + "\"");
logger.info("Processing \"" + file + "\"");
Document doc = DOMBuilder.parse(new FileReader(file), false, true, true);
XMLApplicationProfile.ValidationResult result = profile.validate(file, doc);
scorePerDoc.put(file, result.score());
Map<String, Guideline.Result> results = result.results();
for (Map.Entry entry : results.entrySet()) {
if (entry.getKey().toString().contains("Date")) {
System.out.println(results.get(entry.getKey()).warnings().toString());
System.out.println(results.get(entry.getKey()).errors().toString());
System.out.println(entry.getKey() + " = " + entry.getValue());
System.out.println("\n");
logger.info("Warnings: " + results.get(entry.getKey()).warnings().toString());
logger.info("Errors: " + results.get(entry.getKey()).errors().toString());
logger.info("Result: " + entry.getKey() + " = " + entry.getValue() + "\n");
}
System.out.println(entry.getKey() + " = " + entry.getValue());
logger.debug(entry.getKey() + " = " + entry.getValue());
}
}
catch(Exception e) {
System.out.println(e.getMessage());
System.out.println(e);
logger.error(e.getMessage());
e.printStackTrace();
}
}
// Individual scores
String printout = scorePerDoc.entrySet().stream().
map(entry -> entry.getValue() + ": " + entry.getKey()).collect(Collectors.joining("\n"));
// Average Score
double FinalScore = scorePerDoc.entrySet().stream().mapToDouble(entry -> entry.getValue()).average().getAsDouble();
System.out.println(printout);
System.out.println("\nValidation Score: "+FinalScore);
OptionalDouble optionalFinalScore = scorePerDoc.values().stream().mapToDouble(aDouble -> aDouble).average();
double finalScore = (optionalFinalScore.isPresent() ? optionalFinalScore.getAsDouble() : -1 );
logger.info(printout);
logger.info("\nValidation Score: " + finalScore);
}
}

25
src/test/resources/log4j2.properties
View File

@ -1,25 +0,0 @@
status = error
dest = err
name = PropertiesConfig
property.filename = target/validator_testing.log
appender.rolling.type = RollingFile
appender.rolling.name = RollingFile
appender.rolling.fileName = ${filename}
appender.rolling.filePattern = target/validator_testing -%d{MM-dd-yy-HH-mm-ss}-%i.log.gz
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = %d %p %C{1.} [%t] %m%n
appender.rolling.policies.type = Policies
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
appender.rolling.policies.time.interval = 2
appender.rolling.policies.time.modulate = true
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
appender.rolling.policies.size.size=100MB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.max = 5
logger.rolling.name = eu.dnetlib.validator2
logger.rolling.additivity = false
logger.rolling.appenderRef.rolling.ref = RollingFile

41
src/test/resources/logback.xml Normal file
View File

@ -0,0 +1,41 @@
<configuration debug="false">
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/> <!-- Default delay: 0ms (zero) -->
<!-- <Appenders>-->
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>logs/ValidationEngine.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>logs/ValidationEngine.%i.log.zip</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>20</maxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>50MB</maxFileSize> <!-- Lower the size to check the "RollingFileAppender" during tests. -->
</triggeringPolicy>
<encoder>
<charset>UTF-8</charset>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS z} [%thread] %-5level %logger{36}.%M\(@%line\) - %msg%n</pattern>
</encoder>
</appender>
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<charset>UTF-8</charset>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS z} [%thread] %highlight(%-5level) %cyan(%logger{36}.%M\(@%line\)) - %msg%n</pattern>
</encoder>
</appender>
<!-- </Appenders>-->
<!-- <Loggers>-->
<root level="debug">
<appender-ref ref="Console" />
</root>
<logger name="eu.dnetlib.validator2" level="trace"/>
<!-- </Loggers>-->
</configuration>