From 020ffc973466bbe76fe709b6276c7e9b27e32109 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 24 Jun 2020 09:51:35 +0200 Subject: [PATCH] code for the path of refereed field for production data --- dhp-workflows/dhp-patch/pom.xml | 44 +++++ .../dnetlib/dhp/patchrefereed/Constants.java | 14 ++ .../dhp/patchrefereed/ResultInstance.java | 28 ++++ .../dhp/patchrefereed/SparkPatchRefereed.java | 158 ++++++++++++++++++ .../SparkPrepareResultInstanceList.java | 125 ++++++++++++++ .../dhp/patchrefereed/patch_parameters.json | 31 ++++ .../dhp/patchrefereed/prepare_parameters.json | 26 +++ .../dhp/patchrefereed/PatchRefereedTest.java | 146 ++++++++++++++++ .../simpleTest/preparedInfo.json | 1 + .../simpleTest/preparedInfo2.json | 0 .../simpleTest/prod_publication.json | 1 + .../patchrefereed/simpleTest/publication.json | 1 + .../simpleTest/publication2.json | 1 + 13 files changed, 576 insertions(+) create mode 100644 dhp-workflows/dhp-patch/pom.xml create mode 100644 dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/Constants.java create mode 100644 dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/ResultInstance.java create mode 100644 dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/SparkPatchRefereed.java create mode 100644 dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/SparkPrepareResultInstanceList.java create mode 100644 dhp-workflows/dhp-patch/src/main/resources/eu/dnetlib/dhp/patchrefereed/patch_parameters.json create mode 100644 dhp-workflows/dhp-patch/src/main/resources/eu/dnetlib/dhp/patchrefereed/prepare_parameters.json create mode 100644 dhp-workflows/dhp-patch/src/test/java/eu/dnetlib/dhp/patchrefereed/PatchRefereedTest.java create mode 100644 dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/preparedInfo.json create mode 100644 dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/preparedInfo2.json create mode 100644 dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/prod_publication.json create mode 100644 dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/publication.json create mode 100644 dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/publication2.json diff --git a/dhp-workflows/dhp-patch/pom.xml b/dhp-workflows/dhp-patch/pom.xml new file mode 100644 index 000000000..3f35d5d3e --- /dev/null +++ b/dhp-workflows/dhp-patch/pom.xml @@ -0,0 +1,44 @@ + + + + dhp-workflows + eu.dnetlib.dhp + 1.2.4-SNAPSHOT + + 4.0.0 + + dhp-patch + + + eu.dnetlib.dhp + dhp-common + 1.2.4-SNAPSHOT + compile + + + com.fasterxml.jackson.core + jackson-databind + compile + + + eu.dnetlib.dhp + dhp-aggregation + 1.2.4-SNAPSHOT + compile + + + org.apache.spark + spark-core_2.11 + compile + + + org.apache.spark + spark-sql_2.11 + compile + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/Constants.java b/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/Constants.java new file mode 100644 index 000000000..fb9957fa6 --- /dev/null +++ b/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/Constants.java @@ -0,0 +1,14 @@ +package eu.dnetlib.dhp.patchrefereed; + +import eu.dnetlib.dhp.schema.oaf.Qualifier; + +public class Constants { + + public static final Qualifier DEFAULT_REFEREED = new Qualifier(); + static{ + DEFAULT_REFEREED.setClassid("0000"); + DEFAULT_REFEREED.setClassname("Unknown"); + DEFAULT_REFEREED.setSchemename("dnet:review_levels"); + DEFAULT_REFEREED.setSchemeid("dnet:review_levels"); + } +} diff --git a/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/ResultInstance.java b/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/ResultInstance.java new file mode 100644 index 000000000..ea52e76c4 --- /dev/null +++ b/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/ResultInstance.java @@ -0,0 +1,28 @@ +package eu.dnetlib.dhp.patchrefereed; + +import eu.dnetlib.dhp.schema.oaf.Instance; + +import java.io.Serializable; +import java.util.List; + +public class ResultInstance implements Serializable { + + private String resultId; + private List instanceList; + + public String getResultId() { + return resultId; + } + + public void setResultId(String resultId) { + this.resultId = resultId; + } + + public List getInstanceList() { + return instanceList; + } + + public void setInstanceList(List instanceList) { + this.instanceList = instanceList; + } +} diff --git a/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/SparkPatchRefereed.java b/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/SparkPatchRefereed.java new file mode 100644 index 000000000..9f9692f8a --- /dev/null +++ b/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/SparkPatchRefereed.java @@ -0,0 +1,158 @@ +package eu.dnetlib.dhp.patchrefereed; + +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.oaf.Instance; +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import eu.dnetlib.dhp.schema.oaf.Result; +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +import java.io.Serializable; +import java.util.List; +import java.util.Optional; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +public class SparkPatchRefereed implements Serializable { + + private static final Logger log = LoggerFactory.getLogger(SparkPatchRefereed.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + SparkAtomicActionJob.class + .getResourceAsStream( + "/eu/dnetlib/dhp/patchrefereed/patch_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + final String preparedInfoPath = parser.get("preparedInfoPath"); + log.info("preparedInfoPath {}: ", preparedInfoPath); + + final String inputPath = parser.get("inputPath"); + log.info("inputPath {}: ", inputPath); + + final String resultClassName = parser.get("resultTableName"); + log.info("resultTableName: {}", resultClassName); + + Class resultClazz = (Class) Class.forName(resultClassName); + + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + patchRefereed( + spark, + inputPath, + preparedInfoPath, + outputPath, + resultClazz); + }); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + private static void patchRefereed(SparkSession spark, + String inputPath, + String prearedInfoPath, + String outputPath, + Class resultClazz) { + + Dataset result = readPath(spark, inputPath, resultClazz); + Dataset ri = readPath(spark, prearedInfoPath, ResultInstance.class); + + result.joinWith(ri, result.col("id").equalTo(ri.col("resultId")), "left") + .map((MapFunction, R>)value->{ + R r = value._1(); + Optional oInstanceList = Optional.ofNullable(value._2()); + List resultInstance = r.getInstance(); + if(oInstanceList.isPresent()){ + List instanceList = oInstanceList.get().getInstanceList(); + resultInstance.forEach(i -> checkEquivalence(instanceList, i)); + } + return r; + },Encoders.bean(resultClazz)) + .write() + .mode(SaveMode.Overwrite).option("compression","gzip").json(outputPath); + + } + + private static void checkEquivalence(List resultInstance, Instance i) { + + for(Instance ri : resultInstance){ + if(update(ri,i)){ + return; + } + } + + } + + //verify collectedfrom hostedby openaccess instancetype url + private static boolean update(Instance ri, Instance i) { + if (ri.getAccessright().equals(i.getAccessright()) && + ri.getCollectedfrom().equals(i.getCollectedfrom()) && + ri.getHostedby().equals(i.getHostedby()) && + ri.getInstancetype().equals(i.getInstancetype()) && + equals(ri.getUrl(),i.getUrl())){ + Optional oRefereed = Optional.ofNullable(i.getRefereed()); + if(!oRefereed.isPresent()){ + if (ri.getRefereed().getClassid().equals("")){ + i.setRefereed(Constants.DEFAULT_REFEREED); + }else { + i.setRefereed(ri.getRefereed()); + } + } + return true; + } + return false; + } + + private static boolean equals(List url, List url1) { + for(String u:url){ + if(!url1.contains(u)){ + return false; + } + } + return true; + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + +} diff --git a/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/SparkPrepareResultInstanceList.java b/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/SparkPrepareResultInstanceList.java new file mode 100644 index 000000000..787757eb4 --- /dev/null +++ b/dhp-workflows/dhp-patch/src/main/java/eu/dnetlib/dhp/patchrefereed/SparkPrepareResultInstanceList.java @@ -0,0 +1,125 @@ +package eu.dnetlib.dhp.patchrefereed; + +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.oaf.*; +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.Serializable; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +public class SparkPrepareResultInstanceList implements Serializable { + + private static final Logger log = LoggerFactory.getLogger(SparkPrepareResultInstanceList.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + SparkAtomicActionJob.class + .getResourceAsStream( + "/eu/dnetlib/dhp/patchrefereed/prepare_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + final String inputPath = parser.get("inputPath"); + log.info("inputPath {}: ", inputPath); + + final String resultClassName = parser.get("resultTableName"); + log.info("resultTableName: {}", resultClassName); + + Class resultClazz = (Class) Class.forName(resultClassName); + + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + getAtomicActions( + spark, + inputPath, + outputPath, + resultClazz); + }); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + private static void getAtomicActions(SparkSession spark, + String inputPath, + String outputPath, + Class resultClazz) { + + Dataset result = readPath(spark, inputPath, resultClazz); + + result.map(r -> { + ResultInstance ri = null; + List instanceList = Optional.ofNullable(r.getInstance()) + .map(instances -> instances + .stream() + .map(instance -> { + Optional oRefereed = Optional.ofNullable(instance.getRefereed()); + if (oRefereed.isPresent()) { + return instance; + } else { + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toList())) + .orElse(null); + + if((instanceList != null ) && instanceList.size() > 0){ + ri = new ResultInstance(); + ri.setResultId(r.getId()); + ri.setInstanceList(instanceList); + } + + return ri; + + },Encoders.bean(ResultInstance.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression","gzip") + .json(outputPath); + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + +} diff --git a/dhp-workflows/dhp-patch/src/main/resources/eu/dnetlib/dhp/patchrefereed/patch_parameters.json b/dhp-workflows/dhp-patch/src/main/resources/eu/dnetlib/dhp/patchrefereed/patch_parameters.json new file mode 100644 index 000000000..969f2fdb5 --- /dev/null +++ b/dhp-workflows/dhp-patch/src/main/resources/eu/dnetlib/dhp/patchrefereed/patch_parameters.json @@ -0,0 +1,31 @@ +[ + { + "paramName":"i", + "paramLongName":"inputPath", + "paramDescription": "the path of the sequencial file to read", + "paramRequired": true + }, + { + "paramName":"tn", + "paramLongName":"resultTableName", + "paramDescription": "the name of the result table we are currently working on", + "paramRequired": true + }, + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true + }, + { + "paramName": "ssm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "true if the spark session is managed, false otherwise", + "paramRequired": false + },{ + "paramName": "pip", + "paramLongName": "preparedInfoPath", + "paramDescription": "dex", + "paramRequired": true +} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-patch/src/main/resources/eu/dnetlib/dhp/patchrefereed/prepare_parameters.json b/dhp-workflows/dhp-patch/src/main/resources/eu/dnetlib/dhp/patchrefereed/prepare_parameters.json new file mode 100644 index 000000000..6c60fa9df --- /dev/null +++ b/dhp-workflows/dhp-patch/src/main/resources/eu/dnetlib/dhp/patchrefereed/prepare_parameters.json @@ -0,0 +1,26 @@ +[ + { + "paramName":"i", + "paramLongName":"inputPath", + "paramDescription": "the path of the sequencial file to read", + "paramRequired": true + }, + { + "paramName":"tn", + "paramLongName":"resultTableName", + "paramDescription": "the name of the result table we are currently working on", + "paramRequired": true + }, + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true + }, + { + "paramName": "ssm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "true if the spark session is managed, false otherwise", + "paramRequired": false + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-patch/src/test/java/eu/dnetlib/dhp/patchrefereed/PatchRefereedTest.java b/dhp-workflows/dhp-patch/src/test/java/eu/dnetlib/dhp/patchrefereed/PatchRefereedTest.java new file mode 100644 index 000000000..6c210bd13 --- /dev/null +++ b/dhp-workflows/dhp-patch/src/test/java/eu/dnetlib/dhp/patchrefereed/PatchRefereedTest.java @@ -0,0 +1,146 @@ +package eu.dnetlib.dhp.patchrefereed; + +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.schema.oaf.Publication; +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class PatchRefereedTest { + + private static final Logger log = LoggerFactory.getLogger(PatchRefereedTest.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static SparkSession spark; + + private static Path workingDir; + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(PatchRefereedTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(PatchRefereedTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(PatchRefereedTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + /** + * There are no new relations to be added. All the possible relations have already been linked with the project in + * the graph + * + * @throws Exception + */ + @Test + public void test1() throws Exception { + + SparkPrepareResultInstanceList + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-inputPath", + getClass().getResource("/eu/dnetlib/dhp/patchrefereed/simpleTest/publication.json").getPath(), + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication", + "-outputPath", workingDir.toString() + "/publication" + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/publication") + .map(item -> OBJECT_MAPPER.readValue(item, ResultInstance.class)); + + Assertions.assertEquals(1, tmp.count()); + + + } + + @Test + public void test2() throws Exception { + + SparkPatchRefereed + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-inputPath", + getClass().getResource("/eu/dnetlib/dhp/patchrefereed/simpleTest/prod_publication.json").getPath(), + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication", + "-outputPath", workingDir.toString() + "/publication", + "-preparedInfoPath",getClass().getResource("/eu/dnetlib/dhp/patchrefereed/simpleTest/preparedInfo.json").getPath() + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/publication") + .map(item -> OBJECT_MAPPER.readValue(item, Publication.class)); + + Assertions.assertEquals(1, tmp.count()); + + org.apache.spark.sql.Dataset verificationDataset = spark + .createDataset(tmp.rdd(), Encoders.bean(Publication.class)); + + verificationDataset.createOrReplaceTempView("dataset"); + + verificationDataset.show(false); + + + } + + @Test + public void test3() throws Exception { + SparkPrepareResultInstanceList + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-inputPath", + getClass().getResource("/eu/dnetlib/dhp/patchrefereed/simpleTest/publication2.json").getPath(), + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication", + "-outputPath", workingDir.toString() + "/publication" + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/publication") + .map(item -> OBJECT_MAPPER.readValue(item, ResultInstance.class)); + + Assertions.assertEquals(1, tmp.count()); + } + + //@Test + + +} diff --git a/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/preparedInfo.json b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/preparedInfo.json new file mode 100644 index 000000000..26d31e664 --- /dev/null +++ b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/preparedInfo.json @@ -0,0 +1 @@ +{"instanceList":[{"accessright":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""},"trust":""},"key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a","value":"ORCID"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""},"trust":""},"value":"2002-01-01"},"distributionlocation":"","hostedby":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""},"trust":""},"key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"instancetype":{"classid":"0004","classname":"Conference object","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"license":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""},"trust":""},"value":""},"processingchargeamount":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""},"trust":""},"value":""},"processingchargecurrency":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""},"trust":""},"value":""},"refereed":{"classid":"","classname":"","schemeid":"","schemename":""},"url":[]}],"resultId":"50|orcid_______::6c6960227e2775a019090d8f33e0e6e5"} \ No newline at end of file diff --git a/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/preparedInfo2.json b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/preparedInfo2.json new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/prod_publication.json b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/prod_publication.json new file mode 100644 index 000000000..c34e1502d --- /dev/null +++ b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/prod_publication.json @@ -0,0 +1 @@ +{"collectedfrom":[{"key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a","value":"ORCID","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:actionset:orcidworks-no-doi","classname":"sysimport:actionset:orcidworks-no-doi","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1574862460176,"id":"50|orcid_______::6c6960227e2775a019090d8f33e0e6e5","originalId":["56426925"],"pid":[],"dateofcollection":"2018-10-22","dateoftransformation":"2019-11-27T13:47:40+00:00","extraInfo":[],"oaiprovenance":null,"author":[{"fullname":"Bunkowski, K.","name":"K.","surname":"Bunkowski","rank":1,"pid":[],"affiliation":[]},{"fullname":"Kalinowski, A.","name":"A.","surname":"Kalinowski","rank":2,"pid":[],"affiliation":[]},{"fullname":"Królikowski, J.","name":"J.","surname":"Królikowski","rank":3,"pid":[],"affiliation":[]},{"fullname":"Kudla, M.","name":"M.","surname":"Kudla","rank":4,"pid":[],"affiliation":[]},{"fullname":"Krzysztof Pozniak","name":"Krzysztof","surname":"Pozniak","rank":5,"pid":[{"value":"0000-0001-5426-1423","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":null,"schemename":null},"dataInfo":null}],"affiliation":[]},{"fullname":"Wrochna, G.","name":"G.","surname":"Wrochna","rank":6,"pid":[],"affiliation":[]}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"","classname":"","schemeid":"","schemename":""},"country":[],"subject":[],"title":[{"value":"Pattern comparator trigger algorithm - Implementation in FPGA","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"relevantdate":[{"value":"2002-01-01","qualifier":{"classid":"issued","classname":"issued","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"description":[],"dateofacceptance":{"value":"2002-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"publisher":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"embargoenddate":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"source":[{"value":"Scopus - Elsevier","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"conference-paper","classname":"conference-paper","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"coverage":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":[],"instance":[{"license":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"accessright":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0004","classname":"Conference object","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"url":[],"distributionlocation":"","collectedfrom":{"key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a","value":"ORCID","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"dateofacceptance":{"value":"2002-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"processingchargeamount":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"processingchargecurrency":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}}],"journal":{"name":"","issnPrinted":"","issnOnline":"","issnLinking":"","ep":"","iss":"","sp":"","vol":"","edition":"","conferenceplace":"","conferencedate":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}} \ No newline at end of file diff --git a/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/publication.json b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/publication.json new file mode 100644 index 000000000..78b67d525 --- /dev/null +++ b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/publication.json @@ -0,0 +1 @@ +{"collectedfrom":[{"key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a","value":"ORCID","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:actionset:orcidworks-no-doi","classname":"sysimport:actionset:orcidworks-no-doi","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1574862460176,"id":"50|orcid_______::6c6960227e2775a019090d8f33e0e6e5","originalId":["56426925"],"pid":[],"dateofcollection":"2018-10-22","dateoftransformation":"2019-11-27T13:47:40+00:00","extraInfo":[],"oaiprovenance":null,"author":[{"fullname":"Bunkowski, K.","name":"K.","surname":"Bunkowski","rank":1,"pid":[],"affiliation":[]},{"fullname":"Kalinowski, A.","name":"A.","surname":"Kalinowski","rank":2,"pid":[],"affiliation":[]},{"fullname":"Królikowski, J.","name":"J.","surname":"Królikowski","rank":3,"pid":[],"affiliation":[]},{"fullname":"Kudla, M.","name":"M.","surname":"Kudla","rank":4,"pid":[],"affiliation":[]},{"fullname":"Krzysztof Pozniak","name":"Krzysztof","surname":"Pozniak","rank":5,"pid":[{"value":"0000-0001-5426-1423","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":null,"schemename":null},"dataInfo":null}],"affiliation":[]},{"fullname":"Wrochna, G.","name":"G.","surname":"Wrochna","rank":6,"pid":[],"affiliation":[]}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"","classname":"","schemeid":"","schemename":""},"country":[],"subject":[],"title":[{"value":"Pattern comparator trigger algorithm - Implementation in FPGA","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"relevantdate":[{"value":"2002-01-01","qualifier":{"classid":"issued","classname":"issued","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"description":[],"dateofacceptance":{"value":"2002-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"publisher":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"embargoenddate":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"source":[{"value":"Scopus - Elsevier","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"conference-paper","classname":"conference-paper","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"coverage":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":[],"instance":[{"license":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"accessright":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0004","classname":"Conference object","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"url":[],"distributionlocation":"","collectedfrom":{"key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a","value":"ORCID","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"dateofacceptance":{"value":"2002-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"processingchargeamount":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"processingchargecurrency":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"refereed":{"classid":"","classname":"","schemeid":"","schemename":""}}],"journal":{"name":"","issnPrinted":"","issnOnline":"","issnLinking":"","ep":"","iss":"","sp":"","vol":"","edition":"","conferenceplace":"","conferencedate":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}} \ No newline at end of file diff --git a/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/publication2.json b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/publication2.json new file mode 100644 index 000000000..af0c64d5a --- /dev/null +++ b/dhp-workflows/dhp-patch/src/test/resources/eu/dnetlib/dhp/patchrefereed/simpleTest/publication2.json @@ -0,0 +1 @@ +{"collectedfrom":[{"key":"10|openaire____::806360c771262b4d6770e7cdf04b5c5a","value":"ORCID","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:actionset:orcidworks-no-doi","classname":"sysimport:actionset:orcidworks-no-doi","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1574862460176,"id":"50|orcid_______::6c6960227e2775a019090d8f33e0e6e5","originalId":["56426925"],"pid":[],"dateofcollection":"2018-10-22","dateoftransformation":"2019-11-27T13:47:40+00:00","extraInfo":[],"oaiprovenance":null,"author":[{"fullname":"Bunkowski, K.","name":"K.","surname":"Bunkowski","rank":1,"pid":[],"affiliation":[]},{"fullname":"Kalinowski, A.","name":"A.","surname":"Kalinowski","rank":2,"pid":[],"affiliation":[]},{"fullname":"Królikowski, J.","name":"J.","surname":"Królikowski","rank":3,"pid":[],"affiliation":[]},{"fullname":"Kudla, M.","name":"M.","surname":"Kudla","rank":4,"pid":[],"affiliation":[]},{"fullname":"Krzysztof Pozniak","name":"Krzysztof","surname":"Pozniak","rank":5,"pid":[{"value":"0000-0001-5426-1423","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":null,"schemename":null},"dataInfo":null}],"affiliation":[]},{"fullname":"Wrochna, G.","name":"G.","surname":"Wrochna","rank":6,"pid":[],"affiliation":[]}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"","classname":"","schemeid":"","schemename":""},"country":[],"subject":[],"title":[{"value":"Pattern comparator trigger algorithm - Implementation in FPGA","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"relevantdate":[{"value":"2002-01-01","qualifier":{"classid":"issued","classname":"issued","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"description":[],"dateofacceptance":{"value":"2002-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"publisher":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"embargoenddate":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"source":[{"value":"Scopus - Elsevier","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"conference-paper","classname":"conference-paper","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"coverage":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":[],"instance":[{"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"dataInfo":null,"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"ZENODO"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk","classname":"sysimport:crosswalk","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014-12-31"},"distributionlocation":"","hostedby":{"dataInfo":null,"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"ZENODO"},"instancetype":{"classid":"0001","classname":"Article","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"license":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk","classname":"sysimport:crosswalk","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"http://creativecommons.org/licenses/by/4.0/legalcode"},"processingchargeamount":null,"processingchargecurrency":null,"refereed":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["https://zenodo.org/record/1675653"]}, {"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"dataInfo":null,"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"ZENODO"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2014-01-01"},"distributionlocation":"","hostedby":{"dataInfo":null,"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"ZENODO"},"instancetype":{"classid":"0038","classname":"Other literature type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"license":null,"processingchargeamount":null,"processingchargecurrency":null,"refereed":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["https://zenodo.org/record/1675653"]}],"journal":{"name":"","issnPrinted":"","issnOnline":"","issnLinking":"","ep":"","iss":"","sp":"","vol":"","edition":"","conferenceplace":"","conferencedate":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}}} \ No newline at end of file