D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 8 Projects 2 Releases 1 Wiki Activity

- Use "KryoSerializer" for Spark and register some "result" classes. 

- Code polishing and cleanup.
This commit is contained in:
Lampros Smyrnaios 2024-01-17 18:08:05 +02:00
parent 32e02247bc
commit c17834dddf
12 changed files with 16 additions and 766 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
dhp-workflows/dhp-continuous-validation/installAndRun.sh
View File

@ -1,38 +0,0 @@
# This script installs and runs the project.
DEFAULT_PROFILE='' # It's the empty profile.
NEWER_VERSIONS_PROFILE='-Pscala-2.12'
CHOSEN_MAVEN_PROFILE=${DEFAULT_PROFILE}
# For error-handling, we cannot use the "set -e" since: it has problems https://mywiki.wooledge.org/BashFAQ/105
# So we have our own function, for use when a single command fails.
handle_error () {
echo -e "\n\n$1\n\n"; exit $2
}
# Change the working directory to the script's directory, when running from another location.
cd "${0%/*}" || handle_error "Could not change-dir to this script's dir!" 1
if [[ $# -eq 0 ]]; then
echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: installAndRun.sh <sparkMaster> <justRun: 0 | 1>"; exit 2
fi
sparkMaster=""
justRun=0
if [[ $# -eq 1 ]]; then
sparkMaster=$1
elif [[ $# -eq 2 ]]; then
sparkMaster=$1
justRun=$2
elif [[ $# -gt 2 ]]; then
echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: installAndRun.sh <sparkMaster> <justRun: 0 | 1>"; exit 3
fi
if [[ justRun -eq 0 ]]; then
mvn clean install ${CHOSEN_MAVEN_PROFILE}
fi
ContinuousValidator
test_parquet_file="./src/test/resources/part-00589-733117df-3822-4fce-bded-17289cc5959a-c000.snappy.parquet"
java -jar ./target/dhp-continuous-validation-1.0.0-SNAPSHOT.jar ${sparkMaster} ${test_parquet_file} 1

1
dhp-workflows/dhp-continuous-validation/pom.xml
View File

@ -14,7 +14,6 @@
<artifactId>dhp-continuous-validation</artifactId>
<!-- The "version" is inherited from the parent module. -->
<dependencies>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>

15
dhp-workflows/dhp-continuous-validation/src/main/java/eu/dnetlib/dhp/continuous_validator/ContinuousValidator.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.dhp.continuous_validator;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import java.util.Optional;
@ -16,7 +17,10 @@ import org.apache.spark.sql.SaveMode;
import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.validator2.validation.StandardValidationResult;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.Guideline;
import eu.dnetlib.validator2.validation.guideline.StandardResult;
import eu.dnetlib.validator2.validation.guideline.openaire.*;
import eu.dnetlib.validator2.validation.utils.TestUtils;
@ -61,6 +65,7 @@ public class ContinuousValidator {
// This is needed to implement a unit test in which the spark session is created in the context of the
// unit test itself rather than inside the spark application"
// Set the parquet input, either a parquet-file or a directory with parquet files.
parquet_file_path = parser.get("parquet_file_path");
if (parquet_file_path == null) {
logger.error("The \"parquet_file_path\" was not retrieved from the parameters file: " + parametersFile);
@ -80,8 +85,8 @@ public class ContinuousValidator {
return;
}
if (!outputPath.endsWith("/"))
outputPath += "/";
if (!outputPath.endsWith(File.separator))
outputPath += File.separator;
logger
.info(
@ -112,11 +117,15 @@ public class ContinuousValidator {
SparkConf conf = new SparkConf();
conf.setAppName(ContinuousValidator.class.getSimpleName());
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
conf.registerKryoClasses(new Class[] {
XMLApplicationProfile.ValidationResult.class, Guideline.Result.class, StandardValidationResult.class,
StandardResult.class
});
String finalParquet_file_path = parquet_file_path;
String finalOutputPath = outputPath;
runWithSparkSession(conf, isSparkSessionManaged, spark -> {
// Use a new instance of Document Builder in each worker, as it is not thread-safe.
MapFunction<Row, XMLApplicationProfile.ValidationResult> validateMapFunction = row -> profile
.validate(

2
dhp-workflows/dhp-continuous-validation/src/main/resources/eu/dnetlib/dhp/continuous_validator/input_continuous_validator_parameters.json
View File

@ -8,7 +8,7 @@
{
"paramName": "prq_file",
"paramLongName": "parquet_file_path",
"paramDescription": "the full path for the parquet file to be processed",
"paramDescription": "the full path of a parquet-file or a directory with parquet files, to be processed",
"paramRequired": true
},
{

6
dhp-workflows/dhp-continuous-validation/src/main/resources/eu/dnetlib/dhp/continuous_validator/oozie_app/workflow.xml
View File

@ -2,8 +2,8 @@
<parameters>
<property>
<name>parquet_file_path</name>
<value>./src/test/resources/part-00589-733117df-3822-4fce-bded-17289cc5959a-c000.snappy.parquet</value>
<description>the full path of the parquet file</description>
<value>/var/lib/dnet/mdstore_PROD/md-7763c517-538d-4aa7-83f8-6096b3ce0d96/md-7763c517-538d-4aa7-83f8-6096b3ce0d96-1702622132535/store</value>
<description>the full path of a parquet-file or a directory with parquet files, to be processed</description>
</property>
<property>
<name>openaire_guidelines</name>
@ -12,7 +12,7 @@
</property>
<property>
<name>outputPath</name>
<value>.</value>
<value>/home/lsmyrnaios/continuous_validation/output</value>
<description>the path of the output-directory where the result-json-files will be stored</description>
</property>
<property>

35
dhp-workflows/dhp-continuous-validation/src/test/java/ReadResultsTest.java
View File

@ -1,35 +0,0 @@
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.List;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import eu.dnetlib.dhp.continuous_validator.ContinuousValidator;
public class ReadResultsTest {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ContinuousValidator.class);
public static void main(String[] args) {
try {
List standardValidationResultList = new Gson()
.fromJson(new BufferedReader(new FileReader(ContinuousValidator.RESULTS_FILE_NAME)), List.class);
if (standardValidationResultList == null)
logger.error("Could not map the json to a \"List\" object.");
else if (standardValidationResultList.isEmpty())
logger.warn("The \"standardValidationResultList\" is empty!");
else
logger.info(standardValidationResultList.toString());
} catch (FileNotFoundException fnfe) {
logger.error("The results-file \"" + ContinuousValidator.RESULTS_FILE_NAME + "\" does not exist!");
} catch (Exception e) {
logger
.error("Error when reading the json-results-file \"" + ContinuousValidator.RESULTS_FILE_NAME + "\"", e);
}
}
}

126
dhp-workflows/dhp-continuous-validation/src/test/java/ValidateTestFiles.java
View File

@ -1,126 +0,0 @@
import static eu.dnetlib.dhp.continuous_validator.ContinuousValidator.TEST_FILES_V4_DIR;
import java.io.BufferedWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.JsonIOException;
import eu.dnetlib.dhp.continuous_validator.ContinuousValidator;
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
import eu.dnetlib.validator2.validation.guideline.openaire.LiteratureGuidelinesV4Profile;
import eu.dnetlib.validator2.validation.utils.TestUtils;
import scala.Option;
public class ValidateTestFiles {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ContinuousValidator.class);
public static final String RESULTS_FILE = "results.json";
public static void main(String[] args) {
if (args.length != 3) {
String errorMsg = "Wrong number of arguments given! PLease run the app like so: java -jar build/libs/continuous-validator-1.0.0-SNAPSHOT.jar <sparkMaster> <parquetFileFullPath> <guidelines>";
System.err.println(errorMsg);
logger.error(errorMsg);
System.exit(1);
}
String sparkMaster = args[0];
logger.info("Will use this Spark master: \"" + sparkMaster + "\".");
String parquetFileFullPath = args[1];
String guidelines = args[2];
logger
.info(
"Will validate the contents of parquetFile: \"" + parquetFileFullPath + "\", against guidelines: \""
+ guidelines + "\".");
SparkConf sparkConf = new SparkConf();
sparkConf.setAppName("Continuous-Validator");
sparkConf.setMaster(sparkMaster); // Run on the Spark Cluster.
sparkConf.set("spark.driver.memory", "4096M");
sparkConf
.set("spark.executor.instances", "4") // 4 executors
.set("spark.executor.cores", "1"); // 1 core per executor
sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
sparkConf.set("spark.rdd.compress", "true");
String appVersion = "1.0.0-SNAPSHOT";
/*
* try { Class<?> klass = Class.forName("eu.dnetlib.continuous_validator.BuildConfig"); appVersion = (String)
* klass.getDeclaredField("version").get(null); if ( logger.isTraceEnabled() )
* logger.trace("The app's version is: " + appVersion); } catch (Exception e) {
* logger.error("Error when acquiring the \"appVersion\"!", e); System.exit(1); }
*/
sparkConf.setJars(new String[] {
"build/libs/continuous-validator-" + appVersion + "-all.jar"
}); // This is the "fat-Jar".
sparkConf.validateSettings();
logger.debug("Spark custom configurations: " + sparkConf.getAll().toString());
LiteratureGuidelinesV4Profile profile = new LiteratureGuidelinesV4Profile();
try (JavaSparkContext sc = new JavaSparkContext(sparkConf)) {
JavaPairRDD<String, String> jprdd = sc.wholeTextFiles(TEST_FILES_V4_DIR);
logger.info("Showing the validation-results..");
// Use a new instance of Document Builder in each worker, as it is not thread-safe.
// The "x._1" is the filename and the "x._2" in the content of the file.
List<XMLApplicationProfile.ValidationResult> validationResultsList = jprdd
.map(
x -> profile
.validate(
x._1,
TestUtils.getDocumentBuilder().parse(IOUtils.toInputStream(x._2, StandardCharsets.UTF_8))))
.collect();
if (validationResultsList.isEmpty()) {
logger.error("The \"validationResultsList\" was empty!");
return;
}
if (logger.isDebugEnabled())
validationResultsList.forEach(vr -> logger.debug(vr.id() + " | score:" + vr.score()));
logger.debug(validationResultsList.toString());
try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(RESULTS_FILE), StandardCharsets.UTF_8)) {
writer.write(new Gson().toJson(validationResultsList));
} catch (Exception e) {
logger
.error(
"Error when writing the \"validationResultsList\" as json into the results-file: "
+ RESULTS_FILE);
return;
}
Option<String> uiWebUrl = sc.sc().uiWebUrl();
if (uiWebUrl.isDefined()) {
logger
.info(
"Waiting 60 seconds, before shutdown, for the user to check the jobs' status at: "
+ uiWebUrl.get());
Thread.sleep(60_000);
} else
logger.info("The \"uiWebUrl\" is not defined, in order to check the jobs' status. Shutting down..");
} catch (JsonIOException jie) {
logger.error("Error when writing the validation results to the json file: " + jie.getMessage());
} catch (Exception e) {
logger.error("Error validating directory: " + TEST_FILES_V4_DIR, e);
}
}
}

52
dhp-workflows/dhp-continuous-validation/src/test/resources/openaireguidelinesV4/01_gv4.xml
View File

@ -1,52 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<resource xmlns="http://namespace.openaire.eu/schema/oaire/" xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:oaire="http://namespace.openaire.eu/schema/oaire/" xmlns:datacite="http://datacite.org/schema/kernel-4"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://namespace.openaire.eu/schema/oaire/ https://www.openaire.eu/schema/repo-lit/4.0/openaire.xsd">
<datacite:titles>
<datacite:title xml:lang="eng">Journal bearings subjected to dynamic loads: the analytical mobility method</datacite:title>
</datacite:titles>
<datacite:creators>
<datacite:creator>
<datacite:creatorName>Flores, Paulo</datacite:creatorName>
</datacite:creator>
<datacite:creator>
<datacite:creatorName>Claro, José Carlos Pimenta</datacite:creatorName>
</datacite:creator>
<datacite:creator>
<datacite:creatorName>Ambrósio, Jorge</datacite:creatorName>
</datacite:creator>
</datacite:creators>
<datacite:contributors>
<datacite:contributor contributorType="HostingInstitution">
<datacite:contributorName>Universidade do Minho</datacite:contributorName>
<datacite:nameIdentifier nameIdentifierScheme="e-mail" schemeURI="mailto:repositorium@sdum.uminho.pt">repositorium@sdum.uminho.pt</datacite:nameIdentifier>
</datacite:contributor>
</datacite:contributors>
<datacite:relatedIdentifiers/>
<datacite:dates>
<datacite:date dateType="Accepted">2005-06-16</datacite:date>
<datacite:date dateType="Issued">2005-06-16</datacite:date>
</datacite:dates>
<dc:language>eng</dc:language>
<oaire:resourceType uri="http://purl.org/coar/resource_type/c_5794" resourceTypeGeneral="literature">conference paper</oaire:resourceType>
<dc:description xml:lang="por">The main purpose of this work is to use the analytical mobility method to analyze journal bearings subjected to dynamic loads, with the intent to include it in a general computational program that has been developed for the dynamic analysis of general mechanical systems. A simple journal bearing subjected to a dynamic load is chosen as a demonstrative example, in order to provide the necessary results for a comprehensive discussion of the methodology presented.</dc:description>
<dc:description xml:lang="por">Fundação para a Ciência e a Tecnologia (FCT)</dc:description>
<dc:description xml:lang="por">Fundo Comunitário Europeu FEDER under project POCTI/EME/2001/38281, entitled ‘Dynamic of Mechanical Systems with joint Clearances and Imperfections’</dc:description>
<dc:format>application/pdf</dc:format>
<datacite:identifier identifierType="Handle">https://hdl.handle.net/1822/18042</datacite:identifier>
<datacite:rights rightsURI="http://purl.org/coar/access_right/c_16ec">restricted access</datacite:rights>
<datacite:subjects>
<datacite:subject>Dynamic bearings</datacite:subject>
<datacite:subject>Hydrodynamic lubrication</datacite:subject>
</datacite:subjects>
<datacite:sizes>
<datacite:size>294443 bytes</datacite:size>
</datacite:sizes>
<oaire:file accessRightsURI="http://purl.org/coar/access_right/c_16ec" mimeType="application/pdf" objectType="fulltext">https://repositorium.sdum.uminho.pt/bitstream/1822/18042/1/CI-2005_02.pdf</oaire:file>
<oaire:citationTitle>IberTrib - 3º Congresso Ibérico de Tribologia</oaire:citationTitle>
<oaire:citationStartPage>1</oaire:citationStartPage>
<oaire:citationEndPage>15</oaire:citationEndPage>
<oaire:citationConferencePlace>Guimarães, Portugal</oaire:citationConferencePlace>
</resource>

59
dhp-workflows/dhp-continuous-validation/src/test/resources/openaireguidelinesV4/oai_mediarep_org_doc_2534.xml
View File

@ -1,59 +0,0 @@
<?xml version='1.0' encoding='UTF-8'?><record xmlns="http://www.openarchives.org/OAI/2.0/">
<header>
<identifier>oai:mediarep.org:doc/2534</identifier>
<datestamp>2020-03-27T11:35:21Z</datestamp>
<setSpec setName="Zeitschrift für Medienwissenschaft">com_doc_568</setSpec>
<setSpec setName="Zeitschriften">com_doc_179</setSpec>
<setSpec>col_doc_1963</setSpec>
<setSpec setName="OpenAIRE">openaire</setSpec>
</header>
<metadata>
<resource xmlns="http://namespace.openaire.eu/schema/oaire/"
xmlns:datacite="http://datacite.org/schema/kernel-4" xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://namespace.openaire.eu/schema/oaire/ https://www.openaire.eu/schema/repo-lit/4.0/openaire.xsd">
<datacite:title>Die autogerechte Stadt (Rückführung)</datacite:title>
<datacite:creators>
<datacite:creator>
<datacite:creatorName>Schmitt, Arne</datacite:creatorName>
</datacite:creator>
</datacite:creators>
<datacite:alternateIdentifiers>
<datacite:alternateIdentifier alternateIdentifierType="URL">https://mediarep.org/handle/doc/2534</datacite:alternateIdentifier>
</datacite:alternateIdentifiers>
<datacite:relatedIdentifiers>
<datacite:relatedIdentifier relatedIdentifierType="ISSN" relationType="IsPartOf">1869-1722</datacite:relatedIdentifier>
</datacite:relatedIdentifiers>
<dc:language>deu</dc:language>
<dc:publisher>diaphanes</dc:publisher>
<datacite:date dateType="Issued">2015</datacite:date>
<resourceType resourceTypeGeneral="literature" uri="http://purl.org/coar/resource_type/c_6501">
journal article
</resourceType>
<dc:description>Bildstrecke ausgewählter Abbildungen aus Hans Bernhard Reichows Buch DIE AUTOGERECHTE STADT.</dc:description>
<dc:format>application/pdf</dc:format>
<datacite:identifier identifierType="DOI">http://dx.doi.org/10.25969/mediarep/1436</datacite:identifier>
<datacite:rights uri="http://purl.org/coar/access_right/c_abf2">
open access
</datacite:rights>
<datacite:subjects>
<datacite:subject>Infrastruktur</datacite:subject>
<datacite:subject>Urbanität</datacite:subject>
<datacite:subject>Mobilität</datacite:subject>
<datacite:subject subjectScheme="DDC">300</datacite:subject>
</datacite:subjects>
<version uri="http://purl.org/coar/version/c_970fb48d4fbd8a85">
VoR
</version>
<file accessRightsURI="http://purl.org/coar/access_right/c_abf2" mimeType="application/pdf" objectType="&#xa; fulltext&#xa; ">https://mediarep.org/bitstream/doc/2534/1/ZfM_12_104-133_Schmitt_Die_autogerechte_Stadt.pdf</file>
<citationTitle>Zeitschrift für Medienwissenschaft</citationTitle>
<citationVolume>7</citationVolume>
<citationIssue>1</citationIssue>
<citationStartPage>104</citationStartPage>
<citationEndPage>113</citationEndPage>
<dcterms:audience>Researchers</dcterms:audience>
<dcterms:audience>Students</dcterms:audience>
</resource>
</metadata>
</record>

251
dhp-workflows/dhp-continuous-validation/src/test/resources/openaireguidelinesV4/v4_literature_all_guidelines_record.xml
View File

@ -1,251 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--This document supports all available OpenAIRE Guidelines for Literature Repositories v4-->
<resource
xmlns:oaire="http://namespace.openaire.eu/schema/oaire/"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:datacite="http://datacite.org/schema/kernel-4"
xmlns="http://namespace.openaire.eu/schema/oaire/"
xsi:schemaLocation="http://namespace.openaire.eu/schema/oaire/ https://www.openaire.eu/schema/repo-lit/4.0/openaire.xsd">
<!--1. Title (M)-->
<datacite:titles>
<datacite:title xml:lang="en-US">
National Institute for Environmental Studies and Center
for Climate System Research Japan
</datacite:title>
<datacite:title xml:lang="en-US" titleType="Subtitle">A survey</datacite:title>
</datacite:titles>
<!--2. Creator (M)-->
<datacite:creators>
<datacite:creator>
<datacite:creatorName nameType="Personal">Evans, R.J.</datacite:creatorName>
<datacite:givenName>Robert</datacite:givenName>
<datacite:familyName>Evans</datacite:familyName>
<datacite:affiliation>Institute of Science and Technology</datacite:affiliation>
<datacite:nameIdentifier nameIdentifierScheme="ORCID" schemeURI="http://orcid.org">
1234-1234-1234-1234
</datacite:nameIdentifier>
</datacite:creator>
</datacite:creators>
<!--3. Contributor (MA)-->
<datacite:contributors>
<datacite:contributor contributorType="Other">
<datacite:contributorName nameType="Organizational">i-5uvLfGD2R.y8fXv442y8o.D6v1</datacite:contributorName>
<datacite:givenName>givenName3</datacite:givenName>
<datacite:familyName>familyName3</datacite:familyName>
<datacite:nameIdentifier nameIdentifierScheme="hK9c28uVD" schemeURI="http://fkPymUoN/">MuDRo.ymY7qG1Or.9Ny</datacite:nameIdentifier>
<datacite:nameIdentifier nameIdentifierScheme="k_Q7K8AbQiwLlreHzyU" schemeURI="http://EaTpassQ/">a1Sp</datacite:nameIdentifier>
<datacite:affiliation>affiliation5</datacite:affiliation>
<datacite:affiliation>affiliation6</datacite:affiliation>
</datacite:contributor>
<datacite:contributor contributorType="Editor">
<datacite:contributorName nameType="Organizational">ZxHQr4TGMeqqEOTbMcNd_EMFvIf</datacite:contributorName>
<datacite:givenName>givenName4</datacite:givenName>
<datacite:familyName>familyName4</datacite:familyName>
<datacite:nameIdentifier nameIdentifierScheme="PWhc-.4-Ock6m2WlzXL4km" schemeURI="http://KTYQUXaS/">nJNWaCv</datacite:nameIdentifier>
<datacite:nameIdentifier nameIdentifierScheme="fZBhNb-U2t" schemeURI="http://BJDgOJde/">RobhZUU3Obaqq-3UlrgxKNe-c0</datacite:nameIdentifier>
<datacite:affiliation>affiliation7</datacite:affiliation>
<datacite:affiliation>affiliation8</datacite:affiliation>
</datacite:contributor>
</datacite:contributors>
<!--4. Funding Reference (MA)-->
<oaire:fundingReferences>
<oaire:fundingReference>
<oaire:funderName>pZT7SjviMp4yodvIG</oaire:funderName>
<oaire:funderIdentifier funderIdentifierType="ISNI">ff238yCNbhv5k5Y8AmbpyzYt</oaire:funderIdentifier>
<oaire:fundingStream>svI_6A</oaire:fundingStream>
<oaire:awardNumber awardURI="http://TNrUwkxq/">E-</oaire:awardNumber>
<oaire:awardTitle>award</oaire:awardTitle>
</oaire:fundingReference>
<oaire:fundingReference>
<oaire:funderName>funderName</oaire:funderName>
<oaire:funderIdentifier funderIdentifierType="Crossref Funder">QwqkD4y</oaire:funderIdentifier>
<oaire:fundingStream>yyADU1SFmiS-meqQ6</oaire:fundingStream>
<oaire:awardNumber awardURI="http://oKladiXr/">JyORkpwzt</oaire:awardNumber>
<oaire:awardTitle>gyg2F9EKT7Gp.mSBU8.Drqf</oaire:awardTitle>
</oaire:fundingReference>
</oaire:fundingReferences>
<!--5. Alternate Identifier (R)-->
<datacite:alternateIdentifiers>
<datacite:alternateIdentifier alternateIdentifierType="DOI">10.1002/chem.201701589</datacite:alternateIdentifier>
<datacite:alternateIdentifier alternateIdentifierType="PMID">PMC5574022</datacite:alternateIdentifier>
</datacite:alternateIdentifiers>
<!--6. Related Identifier (R)-->
<datacite:relatedIdentifiers>
<datacite:relatedIdentifier relatedIdentifierType="ISSN" relationType="HasMetadata" relatedMetadataScheme="sdfhr" schemeURI="http://fsdfg.cm" schemeType="XSD" resourceTypeGeneral="DataPaper">
0947-6539
</datacite:relatedIdentifier>
<datacite:relatedIdentifier relatedIdentifierType="EISSN" relationType="IsPartOf" resourceTypeGeneral="Model">1521-3765</datacite:relatedIdentifier>
</datacite:relatedIdentifiers>
<datacite:dates>
<!--7. Embargo Period Date (MA)-->
<datacite:date dateType="Accepted">2011-12-01</datacite:date>
<datacite:date dateType="Available">2012-12-31</datacite:date>
<!--10. Publication Date (M)-->
<datacite:date dateType="Issued">2011</datacite:date>
</datacite:dates>
<!--8. Language (MA)-->
<dc:language>eng</dc:language>
<dc:language>deu</dc:language>
<dc:language>nld</dc:language>
<!--9. Publisher (MA)-->
<dc:publisher>
Loughborough University. Department of Computer Science
</dc:publisher>
<dc:publisher>John Wiley &amp; Sons, Inc. (US)</dc:publisher>
<!--11. Resource Type (M)-->
<oaire:resourceType resourceTypeGeneral="literature" uri="http://purl.org/coar/resource_type/c_93fc">report</oaire:resourceType>
<!--12. Description (MA)-->
<dc:description xml:lang="en-US">
Foreword [by] Hazel Anderson; Introduction; The scientific heresy:
transformation of a society; Consciousness as causal reality [etc]
</dc:description>
<dc:description xml:lang="en-US">
A number of problems in quantum state and system identification are
addressed.
</dc:description>
<!--13. Format (R)-->
<dc:format>video/quicktime</dc:format>
<dc:format>application/pdf</dc:format>
<dc:format>application/xml</dc:format>
<!--14. Resource Identifier (M)-->
<datacite:identifier identifierType="URN">http://urn.kb.se/resolve?urn=urn:nbn:se:uu:diva-160648</datacite:identifier>
<!--15. Access Rights (M)-->
<datacite:rights uri="http://purl.org/coar/access_right/c_f1cf">embargoed access</datacite:rights>
<!--16. Source (R)-->
<dc:source>Ecology Letters (1461023X) vol.4 (2001)</dc:source>
<dc:source>Ecology Letters (1461025X) vol.5 (2002)</dc:source>
<!--17. Subject (MA)-->
<datacite:subjects>
<datacite:subject subjectScheme="fAOT3oCMLqANRdqCniD" schemeURI="http://bsOyHkIY/" valueURI="http://ZiIiSVAz/" xml:lang="it-IT">ft12Zy</datacite:subject>
<datacite:subject subjectScheme="b-FUtIiLRu6RsuIcde3KKhzz.9" schemeURI="http://ttXSOOlB/" valueURI="http://GixLBNfF/"
xml:lang="el-GR">Bf6AElMCkh.mqutOmETp0</datacite:subject>
</datacite:subjects>
<!--18. License Condition (R)-->
<oaire:licenseCondition startDate="2019-02-01" uri="http://creativecommons.org/licenses/by-nc/4.0/">Creative Commons Attribution-NonCommercial</oaire:licenseCondition>
<!--19. Coverage (R)-->
<dc:coverage>2000-2010</dc:coverage>
<dc:coverage>
scheme=historic; content=Ming Dynasty
</dc:coverage>
<!--20. Size (O)-->
<datacite:sizes>
<datacite:size>15 pages</datacite:size>
<datacite:size>6 MB</datacite:size>
</datacite:sizes>
<!--21. Geo Location (O)-->
<datacite:geoLocations>
<datacite:geoLocation>
<datacite:geoLocationPlace>Atlantic Ocean</datacite:geoLocationPlace>
<datacite:geoLocationPoint>
<datacite:pointLongitude>31.233</datacite:pointLongitude>
<datacite:pointLatitude>-67.302</datacite:pointLatitude>
</datacite:geoLocationPoint>
<datacite:geoLocationBox>
<datacite:westBoundLongitude>-71.032</datacite:westBoundLongitude>
<datacite:eastBoundLongitude>-68.211</datacite:eastBoundLongitude>
<datacite:southBoundLatitude>41.090</datacite:southBoundLatitude>
<datacite:northBoundLatitude>42.893</datacite:northBoundLatitude>
</datacite:geoLocationBox>
<datacite:geoLocationPolygon>
<datacite:polygonPoint>
<datacite:pointLongitude>88.557</datacite:pointLongitude>
<datacite:pointLatitude>-0.604</datacite:pointLatitude>
</datacite:polygonPoint>
<datacite:polygonPoint>
<datacite:pointLongitude>-143.373</datacite:pointLongitude>
<datacite:pointLatitude>88.832</datacite:pointLatitude>
</datacite:polygonPoint>
<datacite:polygonPoint>
<datacite:pointLongitude>87.3</datacite:pointLongitude>
<datacite:pointLatitude>-36.556</datacite:pointLatitude>
</datacite:polygonPoint>
<datacite:polygonPoint>
<datacite:pointLongitude>129.128</datacite:pointLongitude>
<datacite:pointLatitude>4.616</datacite:pointLatitude>
</datacite:polygonPoint>
<datacite:inPolygonPoint>
<datacite:pointLongitude>-17.547</datacite:pointLongitude>
<datacite:pointLatitude>-47.629</datacite:pointLatitude>
</datacite:inPolygonPoint>
</datacite:geoLocationPolygon>
<datacite:geoLocationPolygon>
<datacite:polygonPoint>
<datacite:pointLongitude>78.121</datacite:pointLongitude>
<datacite:pointLatitude>19.341</datacite:pointLatitude>
</datacite:polygonPoint>
<datacite:polygonPoint>
<datacite:pointLongitude>-118.035</datacite:pointLongitude>
<datacite:pointLatitude>53.647</datacite:pointLatitude>
</datacite:polygonPoint>
<datacite:polygonPoint>
<datacite:pointLongitude>-49.07</datacite:pointLongitude>
<datacite:pointLatitude>-45.561</datacite:pointLatitude>
</datacite:polygonPoint>
<datacite:polygonPoint>
<datacite:pointLongitude>132.484</datacite:pointLongitude>
<datacite:pointLatitude>-41.146</datacite:pointLatitude>
</datacite:polygonPoint>
<datacite:inPolygonPoint>
<datacite:pointLongitude>179.293</datacite:pointLongitude>
<datacite:pointLatitude>15.364</datacite:pointLatitude>
</datacite:inPolygonPoint>
</datacite:geoLocationPolygon>
</datacite:geoLocation>
</datacite:geoLocations>
<!--22. Resource Version (R)-->
<oaire:version uri="http://purl.org/coar/version/c_be7fb7dd8ff6fe43">NA</oaire:version>
<!--23. File Location (MA)-->
<oaire:file accessRightsURI="http://purl.org/coar/access_right/c_abf2" mimeType="application/pdf" objectType="fulltext">http://link-to-the-fulltext.org</oaire:file>
<oaire:file accessRightsURI="http://purl.org/coar/access_right/c_abf2" mimeType="application/pdf" objectType="fulltext">http://europepmc.org/articles/PMC5574022?pdf=render</oaire:file>
<!--24. Citation Title (R)-->
<oaire:citationTitle>some Journal Title</oaire:citationTitle>
<!--25. Citation Volume (R)-->
<oaire:citationVolume>10</oaire:citationVolume>
<!--26. Citation Issue (R)-->
<oaire:citationIssue>1</oaire:citationIssue>
<!--27. Citation Start Page (R)-->
<oaire:citationStartPage>100</oaire:citationStartPage>
<!--28. Citation End Page (R)-->
<oaire:citationEndPage>105</oaire:citationEndPage>
<!--29. Citation Edition (R)-->
<oaire:citationEdition>2</oaire:citationEdition>
<!--30. Citation Conference Place (R)-->
<oaire:citationConferencePlace>Berlin</oaire:citationConferencePlace>
<!--31. Citation Conference Date (R)-->
<oaire:citationConferenceDate>2013-10-22 - 2013-10-23</oaire:citationConferenceDate>
<!--32. Audience (O)-->
<dcterms:audience>Researchers</dcterms:audience>
<dcterms:audience>Students</dcterms:audience>
</resource>

197
dhp-workflows/dhp-continuous-validation/src/test/resources/openaireguidelinesV4/v4_literature_all_invalid_guidelines_record.xml
View File

@ -1,197 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--This document supports all available OpenAIRE Guidelines for Literature Repositories v4-->
<resource
xmlns:oaire="http://namespace.openaire.eu/schema/oaire/"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:datacite="http://datacite.org/schema/kernel-4"
xmlns="http://namespace.openaire.eu/schema/oaire/"
xsi:schemaLocation="http://namespace.openaire.eu/schema/oaire/ https://www.openaire.eu/schema/repo-lit/4.0/openaire.xsd">
<!--1. Title (M)-->
<datacite:titles>
<datacite:notitle xml:lang="en-US">
National Institute for Environmental Studies and Center
for Climate System Research Japan
</datacite:notitle>
</datacite:titles>
<!--2. Creator (M)-->
<datacite:creators>
<datacite:creator>
<!-- <datacite:creatorName nameType="Personal">Evans, R.J.</datacite:creatorName>-->
<!-- <datacite:givenName>Robert</datacite:givenName>-->
<!-- <datacite:familyName>Evans</datacite:familyName>-->
<!-- <datacite:affiliation>Institute of Science and Technology</datacite:affiliation>-->
<!-- <datacite:nameIdentifier nameIdentifierScheme="ORCID" schemeURI="http://orcid.org">-->
<!-- 1234-1234-1234-1234-->
<!-- </datacite:nameIdentifier>-->
</datacite:creator>
</datacite:creators>
<!--3. Contributor (MA)-->
<datacite:contributors>
<datacite:contributor>
<!-- <datacite:contributorName nameType="Organizational">i-5uvLfGD2R.y8fXv442y8o.D6v1</datacite:contributorName>-->
<!-- <datacite:givenName>givenName3</datacite:givenName>-->
<!-- <datacite:familyName>familyName3</datacite:familyName>-->
<datacite:nameIdentifier>MuDRo.ymY7qG1Or.9Ny</datacite:nameIdentifier>
<datacite:nameIdentifier nameIdentifierScheme="k_Q7K8AbQiwLlreHzyU" schemeURI="http://EaTpassQ/">a1Sp</datacite:nameIdentifier>
<datacite:affiliation>affiliation5</datacite:affiliation>
<datacite:affiliation>affiliation6</datacite:affiliation>
</datacite:contributor>
</datacite:contributors>
<!--4. Funding Reference (MA)-->
<oaire:fundingReferences>
<oaire:fundingReference>
<!-- <oaire:funderName>pZT7SjviMp4yodvIG</oaire:funderName> 1 error for missing mandatory element-->
<oaire:funderIdentifier funderIdentifierType="invalid">ff238yCNbhv5k5Y8AmbpyzYt</oaire:funderIdentifier><!--1 warning for incorrect value of funderIdentifierType-->
<oaire:fundingStream>svI_6A</oaire:fundingStream>
<oaire:awardNumber>E-</oaire:awardNumber><!--1 warning for missing awardURI attribute-->
<!--1 warning for missing oaire:awardTitle element-->
</oaire:fundingReference>
<oaire:fundingReference>
<oaire:funderName>funderName</oaire:funderName>
<oaire:funderIdentifier funderIdentifierType="Crossref Funder">QwqkD4y</oaire:funderIdentifier>
<oaire:fundingStream>yyADU1SFmiS-meqQ6</oaire:fundingStream>
<oaire:awardNumber awardURI="http://oKladiXr/">JyORkpwzt</oaire:awardNumber>
<oaire:awardTitle>gyg2F9EKT7Gp.mSBU8.Drqf</oaire:awardTitle>
</oaire:fundingReference>
</oaire:fundingReferences>
<!--5. Alternate Identifier (R)-->
<datacite:alternateIdentifiers>
<datacite:alternateIdentifier alternateIdentifierType="invalid">10.1002/chem.201701589</datacite:alternateIdentifier><!--1 error for incorrect value of alternateIdentifierType-->
<datacite:alternateIdentifier alternateIdentifierType="PMID">PMC5574022</datacite:alternateIdentifier>
</datacite:alternateIdentifiers>
<!--6. Related Identifier (R)-->
<datacite:relatedIdentifiers>
<datacite:relatedIdentifier relatedIdentifierType="ISSN" relationType="HasMetadata" relatedMetadataScheme="sdfhr" schemeURI="http://fsdfg.cm" schemeType="XSD" resourceTypeGeneral="DataPaper">
0947-6539
</datacite:relatedIdentifier>
<datacite:relatedIdentifier relatedIdentifierType="invalid" relationType="invalid" resourceTypeGeneral="Model">1521-3765</datacite:relatedIdentifier><!--2 errors for incorrect value of relationType and relatedIdentifierType-->
</datacite:relatedIdentifiers>
<datacite:dates>
<!--7. Embargo Period Date (MA)-->
<datacite:nodate dateType="Accepted">2011-12-01</datacite:nodate>
<datacite:nodate dateType="Available">2012-12-01</datacite:nodate>
<!--10. Publication Date (M)-->
<datacite:date dateType="invalid">2011</datacite:date>
</datacite:dates>
<!--8. Language (MA)-->
<dc:language>eng</dc:language>
<dc:language>deu</dc:language>
<dc:language>invalidTag</dc:language><!--1 error for not allowed value-->
<!--9. Publisher (MA)-->
<dc:nopublisher>
Loughborough University. Department of Computer Science
</dc:nopublisher>
<dc:nopublisher>John Wiley &amp; Sons, Inc. (US)</dc:nopublisher>
<!--11. Resource Type (M)-->
<oaire:resourceType resourceTypeGeneral="invalid" uri="http://invalid">report</oaire:resourceType><!--2 errors for invalid mandatory attributes-->
<!--12. Description (MA)-->
<dc:description xml:lang="invalidTag"><!--1 warning for not allowed value of optional attribute-->
Foreword [by] Hazel Anderson; Introduction; The scientific heresy:
transformation of a society; Consciousness as causal reality [etc]
</dc:description>
<dc:description xml:lang="en-US">
A number of problems in quantum state and system identification are
addressed.
</dc:description>
<!--13. Format (R)-->
<dc:format>video/quicktime</dc:format>
<dc:format>application/pdf</dc:format>
<dc:format>application/invalid</dc:format><!--Should return 1 warning for not allowed value?-->
<!--14 . Resource Identifier (M)-->
<datacite:identifier identifierType="invalid">http://urn.kb.se/resolve?urn=urn:nbn:se:uu:diva-160648</datacite:identifier><!--1 error for invalid mandatory attribute-->
<!--15. Access Rights (M)-->
<datacite:rights uri="http://purl.org/coar/access_right/c_f1cf">invalid</datacite:rights><!--1 error for not allowed value-->
<!--16. Source (R)-->
<dc:nosource>Ecology Letters (1461023X) vol.4 (2001)</dc:nosource>
<dc:nosource>Ecology Letters (1461025X) vol.5 (2002)</dc:nosource>
<!--17. Subject (MA)-->
<datacite:subjects>
<datacite:subject>ft12Zy</datacite:subject><!--3 warnings for missing attributes-->
<datacite:subject subjectScheme="b-FUtIiLRu6RsuIcde3KKhzz.9" schemeURI="http://ttXSOOlB/" valueURI="http://GixLBNfF/"
xml:lang="el-GR">Bf6AElMCkh.mqutOmETp0</datacite:subject>
</datacite:subjects>
<!--18. License Condition (R)-->
<oaire:licenseCondition startDate="invalid date">Creative Commons Attribution-NonCommercial</oaire:licenseCondition><!--2 errors for invalid attributes-->
<!--19. Coverage (R)-->
<dc:nocoverage>2000-2010</dc:nocoverage>
<dc:nocoverage>
scheme=historic; content=Ming Dynasty
</dc:nocoverage>
<!--20. Size (O)-->
<datacite:sizes>
<!-- <datacite:size>15 pages</datacite:size>-->
<!-- <datacite:size>6 MB</datacite:size>-->
</datacite:sizes>
<!--21. Geo Location (O)-->
<datacite:geoLocations>
<datacite:geoLocation>
<datacite:geoLocationPlace>Atlantic Ocean</datacite:geoLocationPlace>
<datacite:geoLocationPoint>
<datacite:pointLongitude>31.233</datacite:pointLongitude>
<datacite:nopointLatitude>-67.302</datacite:nopointLatitude><!--1 error-->
</datacite:geoLocationPoint>
<datacite:geoLocationBox>
<datacite:westBoundLongitude>-71.032</datacite:westBoundLongitude>
<datacite:eastBoundLongitude>-68.211</datacite:eastBoundLongitude>
<datacite:nosouthBoundLatitude>41.090</datacite:nosouthBoundLatitude><!--1 error-->
<datacite:northBoundLatitude>42.893</datacite:northBoundLatitude>
</datacite:geoLocationBox>
</datacite:geoLocation>
</datacite:geoLocations>
<!--22. Resource Version (R)-->
<oaire:version>irrelevant</oaire:version><!--1 warning for missing MA attribute-->
<!--23. File Location (MA)-->
<oaire:file accessRightsURI="http://invalid" mimeType="invalid/type" objectType="invalid">http://link-to-the-fulltext.org</oaire:file><!--3 warnings for incorrect attributes-->
<oaire:file accessRightsURI="http://purl.org/coar/access_right/c_abf2" mimeType="application/pdf" objectType="fulltext">http://europepmc.org/articles/PMC5574022?pdf=render</oaire:file>
<!--24. Citation Title (R)-->
<oaire:nocitationTitle>some Journal Title</oaire:nocitationTitle>
<!--25. Citation Volume (R)-->
<oaire:nocitationVolume>10</oaire:nocitationVolume>
<!--26. Citation Issue (R)-->
<oaire:nocitationIssue>1</oaire:nocitationIssue>
<!--27. Citation Start Page (R)-->
<oaire:nocitationStartPage>100</oaire:nocitationStartPage>
<!--28. Citation End Page (R)-->
<oaire:nocitationEndPage>105</oaire:nocitationEndPage>
<!--29. Citation Edition (R)-->
<oaire:nocitationEdition>2</oaire:nocitationEdition>
<!--30. Citation Conference Place (R)-->
<oaire:nocitationConferencePlace>Berlin</oaire:nocitationConferencePlace>
<!--31. Citation Conference Date (R)-->
<oaire:citationConferenceDate>2013-02-29</oaire:citationConferenceDate><!--1 warning for not allowed value-->
<!--32. Audience (O)-->
<dcterms:noaudience>Researchers</dcterms:noaudience>
<dcterms:noaudience>Students</dcterms:noaudience>
</resource>

BIN
dhp-workflows/dhp-continuous-validation/src/test/resources/part-00589-733117df-3822-4fce-bded-17289cc5959a-c000.snappy.parquet
View File

Binary file not shown.