From 8b2043c7b1924d47cc9d5871649929ca0c437ff8 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 14 Apr 2020 16:43:40 +0200 Subject: [PATCH 01/88] introducing List generic container for Relation specific properties. Ref ticket https://issue.openaire.research-infrastructures.eu/issues/5512 --- .../eu/dnetlib/dhp/schema/oaf/Relation.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java index 6871c0197..3b45c4f7d 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java @@ -6,20 +6,49 @@ import java.util.stream.Stream; import static com.google.common.base.Preconditions.checkArgument; +/** + * Relation aims to model any edge between two nodes in the OpenAIRE graph. It has a source id and a target id + * pointing to graph node identifiers and it is further characterised by the semantic of the link throught the fields + * relType, subRelType and relClass. Provenance information is modeled according to the dataInfo element and collectedFrom, + * while individual relationship types can provide extra information via the properties field. + */ public class Relation extends Oaf { + /** + * Main relationship classifier, values include 'resultResult', 'resultProject', 'resultOrganization', etc. + */ private String relType; + /** + * Further classifies a relationship, values include 'affiliation', 'similarity', 'supplement', etc. + */ private String subRelType; + /** + * Indicates the direction of the relationship, values include 'isSupplementTo', 'isSupplementedBy', 'merges, 'isMergedIn'. + */ private String relClass; + /** + * The source entity id. + */ private String source; + /** + * The target entity id. + */ private String target; + /** + * The list of datasource id/name pairs providing this relationship. + */ private List collectedFrom = new ArrayList<>(); + /** + * List of relation specific properties. Values values include 'similarityLevel', indicating the similarity score between a pair of publications. + */ + private List properties = new ArrayList<>(); + public String getRelType() { return relType; } @@ -68,6 +97,14 @@ public class Relation extends Oaf { this.collectedFrom = collectedFrom; } + public List getProperties() { + return properties; + } + + public void setProperties(List properties) { + this.properties = properties; + } + public void mergeFrom(final Relation r) { checkArgument(Objects.equals(getSource(), r.getSource()),"source ids must be equal"); From cc67dbff819594b3b05a3eedefe05a058bcfb15d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 14 Apr 2020 17:11:55 +0200 Subject: [PATCH 02/88] typo in text --- .../src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java index 3b45c4f7d..d8d150831 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java @@ -45,7 +45,7 @@ public class Relation extends Oaf { private List collectedFrom = new ArrayList<>(); /** - * List of relation specific properties. Values values include 'similarityLevel', indicating the similarity score between a pair of publications. + * List of relation specific properties. Values include 'similarityLevel', indicating the similarity score between a pair of publications. */ private List properties = new ArrayList<>(); From 11938dac5e39ba083842e6cd2401e1ddee29a196 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 4 May 2020 16:47:07 +0200 Subject: [PATCH 03/88] this commit adds: validated/validationDate to relationships; measure type and simple unit test to indicate the relative serialization --- .../eu/dnetlib/dhp/schema/oaf/Measure.java | 52 +++++++++++++++++++ .../eu/dnetlib/dhp/schema/oaf/Relation.java | 40 ++++++++++++++ .../eu/dnetlib/dhp/schema/oaf/Result.java | 12 +++++ .../dnetlib/dhp/schema/oaf/MeasureTest.java | 37 +++++++++++++ 4 files changed, 141 insertions(+) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java index 98adffd2a..c37e76061 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java @@ -1,4 +1,56 @@ package eu.dnetlib.dhp.schema.oaf; +import com.google.common.base.Objects; + +import java.util.List; + +/** + * Represent a measure, must be further described by a system available resource providing name and descriptions. + */ public class Measure { + + /** + * Unique measure identifier. + */ + private String id; + + /** + * List of units associated with this measure. KeyValue provides a pair to store the laber (key) and the value, + * plus common provenance information. + */ + private List unit; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public List getUnit() { + return unit; + } + + public void setUnit(List unit) { + this.unit = unit; + } + + public void mergeFrom(Measure m) { + //TODO + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Measure measure = (Measure) o; + return Objects.equal(id, measure.id) && + Objects.equal(unit, measure.unit); + } + + @Override + public int hashCode() { + return Objects.hashCode(id, unit); + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java index 76503f885..d77bd7d73 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java @@ -40,6 +40,21 @@ public class Relation extends Oaf { */ private String target; + /** + * Was this relationship authoritatively validated? + */ + private Boolean validated; + + /** + * When was this relationship authoritatively validated. + */ + private String validationDate; + + /** + * List of relation specific properties. Values include 'similarityLevel', indicating the similarity score between a pair of publications. + */ + private List properties = new ArrayList<>(); + public String getRelType() { return relType; } @@ -80,6 +95,30 @@ public class Relation extends Oaf { this.target = target; } + public List getProperties() { + return properties; + } + + public void setProperties(List properties) { + this.properties = properties; + } + + public Boolean getValidated() { + return validated; + } + + public void setValidated(Boolean validated) { + this.validated = validated; + } + + public String getValidationDate() { + return validationDate; + } + + public void setValidationDate(String validationDate) { + this.validationDate = validationDate; + } + public void mergeFrom(final Relation r) { checkArgument(Objects.equals(getSource(), r.getSource()), "source ids must be equal"); @@ -122,4 +161,5 @@ public class Relation extends Oaf { public int hashCode() { return Objects.hash(relType, subRelType, relClass, source, target, collectedfrom); } + } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java index 711b1ca68..11316f36e 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java @@ -7,6 +7,8 @@ import java.util.List; public class Result extends OafEntity implements Serializable { + private List measures; + private List author; // resulttype allows subclassing results into publications | datasets | software @@ -51,6 +53,14 @@ public class Result extends OafEntity implements Serializable { private List instance; + public List getMeasures() { + return measures; + } + + public void setMeasures(List measures) { + this.measures = measures; + } + public List getAuthor() { return author; } @@ -229,6 +239,8 @@ public class Result extends OafEntity implements Serializable { Result r = (Result) e; + //TODO consider merging also Measures + instance = mergeLists(instance, r.getInstance()); if (r.getBestaccessright() != null && compareTrust(this, r) < 0) diff --git a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java index 26376349c..25d929db2 100644 --- a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java +++ b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java @@ -1,4 +1,41 @@ package eu.dnetlib.dhp.schema.oaf; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + public class MeasureTest { + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + + @Test + public void testMeasureSerialization() throws IOException { + + Measure m = new Measure(); + + m.setId("popularity"); + m.setUnit(Lists.newArrayList( + unit("score", "0.5"))); + + String s = OBJECT_MAPPER.writeValueAsString(m); + System.out.println(s); + + Measure mm = OBJECT_MAPPER.readValue(s, Measure.class); + + Assertions.assertNotNull(mm); + } + + private KeyValue unit(String key, String value) { + KeyValue unit = new KeyValue(); + unit.setKey(key); + unit.setValue(value); + return unit; + } + } From c54d7ca18c1c4a4f8fe64b05d94d3e74fb25b4a0 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 4 May 2020 17:02:40 +0200 Subject: [PATCH 04/88] example measures in serialization test --- .../dnetlib/dhp/schema/oaf/MeasureTest.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java index 25d929db2..4275e2c56 100644 --- a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java +++ b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java @@ -1,13 +1,14 @@ package eu.dnetlib.dhp.schema.oaf; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.List; public class MeasureTest { @@ -17,16 +18,22 @@ public class MeasureTest { @Test public void testMeasureSerialization() throws IOException { - Measure m = new Measure(); - - m.setId("popularity"); - m.setUnit(Lists.newArrayList( + Measure popularity = new Measure(); + popularity.setId("popularity"); + popularity.setUnit(Lists.newArrayList( unit("score", "0.5"))); + Measure influence = new Measure(); + influence.setId("influence"); + influence.setUnit(Lists.newArrayList( + unit("score", "0.3"))); + + List m = Lists.newArrayList(popularity, influence); + String s = OBJECT_MAPPER.writeValueAsString(m); System.out.println(s); - Measure mm = OBJECT_MAPPER.readValue(s, Measure.class); + List mm = OBJECT_MAPPER.readValue(s, new TypeReference>() { }); Assertions.assertNotNull(mm); } From 22cb9e0da7ce016853c741017f98ca300e4197ab Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 15 May 2020 18:18:01 +0200 Subject: [PATCH 05/88] simple code to get file from URL --- .../eu/dnetlib/dhp/schema/oaf/Programme.java | 4 + .../dhp/actionset/h2020programme/GetFile.java | 53 +++++++++ .../h2020programme/action_set_parameters.json | 0 .../oozie_app/lib/scripts/getprogrammefile.sh | 0 .../oozie_app/lib/scripts/getprojectfile.sh | 0 .../h2020programme/oozie_app/workflow.xml | 112 ++++++++++++++++++ .../actionset/h2020programme/parameters.json | 86 ++++++++++++++ 7 files changed, 255 insertions(+) create mode 100644 dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionset/h2020programme/GetFile.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/action_set_parameters.json create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprogrammefile.sh create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprojectfile.sh create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java new file mode 100644 index 000000000..f91333616 --- /dev/null +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java @@ -0,0 +1,4 @@ +package eu.dnetlib.dhp.schema.oaf; + +public class Programme { +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionset/h2020programme/GetFile.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionset/h2020programme/GetFile.java new file mode 100644 index 000000000..2fed1a0e3 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionset/h2020programme/GetFile.java @@ -0,0 +1,53 @@ + +package eu.dnetlib.dhp.actionset.h2020programme; + +import java.io.*; +import java.net.URL; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class GetFile { + + private static final Log log = LogFactory.getLog(GetFile.class); + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + GetFile.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionset/h2020programme/parameters.json"))); + + Configuration conf = new Configuration(); + + parser.parseArgument(args); + + final String fileURL = parser.get("fileURL"); + final String hdfsPath = parser.get("hdfsPath"); + final String hdfsNameNode = parser.get("hdfsNameNode"); + + conf.set("fs.defaultFS", hdfsNameNode); + FileSystem fileSystem = FileSystem.get(conf); + Path hdfsWritePath = new Path(hdfsPath); + FSDataOutputStream fsDataOutputStream = null; + if (fileSystem.exists(hdfsWritePath)) { + fsDataOutputStream = fileSystem.append(hdfsWritePath); + } else { + fsDataOutputStream = fileSystem.create(hdfsWritePath); + } + + InputStream is = new BufferedInputStream(new URL(fileURL).openStream()); + + org.apache.hadoop.io.IOUtils.copyBytes(is, fsDataOutputStream, 4096, true); + + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/action_set_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/action_set_parameters.json new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprogrammefile.sh b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprogrammefile.sh new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprojectfile.sh b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprojectfile.sh new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml new file mode 100644 index 000000000..3e7f68401 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml @@ -0,0 +1,112 @@ + + + + sequenceFilePath + the path to store the sequence file of the native metadata collected + + + + mdStorePath + the path of the native mdstore + + + + apiDescription + A json encoding of the API Description class + + + + dataSourceInfo + A json encoding of the Datasource Info + + + identifierPath + An xpath to retrieve the metadata idnentifier for the generation of DNet Identifier + + + + metadataEncoding + The type of the metadata XML/JSON + + + + timestamp + The timestamp of the collection date + + + + workflowId + The identifier of the workflow + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + ${jobTracker} + ${nameNode} + eu.dnetlib.dhp.collection.worker.DnetCollectorWorker + -p${sequenceFilePath} + -a${apiDescription} + -n${nameNode} + -rh${rmq_host} + -ru${rmq_user} + -rp${rmq_pwd} + -rr${rmq_report} + -ro${rmq_ongoing} + -usandro.labruzzo + -w${workflowId} + + + + + + + ${jobTracker} + ${nameNode} + yarn + cluster + GenerateNativeStoreSparkJob + eu.dnetlib.dhp.collection.GenerateNativeStoreSparkJob + dhp-aggregations-1.0.0-SNAPSHOT.jar + --num-executors 50 --conf spark.yarn.jars="hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2" + --encoding ${metadataEncoding} + --dateOfCollection ${timestamp} + --provenance ${dataSourceInfo} + --xpath${identifierPath} + --input${sequenceFilePath} + --output${mdStorePath} + -rh${rmq_host} + -ru${rmq_user} + -rp${rmq_pwd} + -rr${rmq_report} + -ro${rmq_ongoing} + -w${workflowId} + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json new file mode 100644 index 000000000..4a6aec5ee --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json @@ -0,0 +1,86 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false + }, + { + "paramName": "e", + "paramLongName": "encoding", + "paramDescription": "the encoding of the input record should be JSON or XML", + "paramRequired": true + }, + { + "paramName": "d", + "paramLongName": "dateOfCollection", + "paramDescription": "the date when the record has been stored", + "paramRequired": true + }, + { + "paramName": "p", + "paramLongName": "provenance", + "paramDescription": "the infos about the provenance of the collected records", + "paramRequired": true + }, + { + "paramName": "x", + "paramLongName": "xpath", + "paramDescription": "the xpath to identify the record identifier", + "paramRequired": true + }, + { + "paramName": "i", + "paramLongName": "input", + "paramDescription": "the path of the sequencial file to read", + "paramRequired": true + }, + { + "paramName": "o", + "paramLongName": "output", + "paramDescription": "the path of the result DataFrame on HDFS", + "paramRequired": true + }, + { + "paramName": "ru", + "paramLongName": "rabbitUser", + "paramDescription": "the user to connect with RabbitMq for messaging", + "paramRequired": true + }, + { + "paramName": "rp", + "paramLongName": "rabbitPassword", + "paramDescription": "the password to connect with RabbitMq for messaging", + "paramRequired": true + }, + { + "paramName": "rh", + "paramLongName": "rabbitHost", + "paramDescription": "the host of the RabbitMq server", + "paramRequired": true + }, + { + "paramName": "ro", + "paramLongName": "rabbitOngoingQueue", + "paramDescription": "the name of the ongoing queue", + "paramRequired": true + }, + { + "paramName": "rr", + "paramLongName": "rabbitReportQueue", + "paramDescription": "the name of the report queue", + "paramRequired": true + }, + { + "paramName": "w", + "paramLongName": "workflowId", + "paramDescription": "the identifier of the dnet Workflow", + "paramRequired": true + }, + { + "paramName": "t", + "paramLongName": "isTest", + "paramDescription": "the name of the report queue", + "paramRequired": false + } +] \ No newline at end of file From 83c262a483a4ecc4d7a12d6ddab2086c2f447387 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 15 May 2020 18:18:31 +0200 Subject: [PATCH 06/88] workflow to download the files --- .../h2020programme/oozie_app/workflow.xml | 145 +++++++----------- 1 file changed, 57 insertions(+), 88 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml index 3e7f68401..9b200c2a9 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml @@ -1,112 +1,81 @@ - + - sequenceFilePath - the path to store the sequence file of the native metadata collected + projectFileURL + the url where to get the projects file - mdStorePath - the path of the native mdstore + programmeFileURL + the url where to get the programme file - apiDescription - A json encoding of the API Description class - - - - dataSourceInfo - A json encoding of the Datasource Info - - - identifierPath - An xpath to retrieve the metadata idnentifier for the generation of DNet Identifier - - - - metadataEncoding - The type of the metadata XML/JSON - - - - timestamp - The timestamp of the collection date - - - - workflowId - The identifier of the workflow + outputPath + path where to store the action set - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - + - - - - + + - + - - - ${jobTracker} - ${nameNode} - eu.dnetlib.dhp.collection.worker.DnetCollectorWorker - -p${sequenceFilePath} - -a${apiDescription} - -n${nameNode} - -rh${rmq_host} - -ru${rmq_user} - -rp${rmq_pwd} - -rr${rmq_report} - -ro${rmq_ongoing} - -usandro.labruzzo - -w${workflowId} - - - - - - - ${jobTracker} - ${nameNode} - yarn - cluster - GenerateNativeStoreSparkJob - eu.dnetlib.dhp.collection.GenerateNativeStoreSparkJob - dhp-aggregations-1.0.0-SNAPSHOT.jar - --num-executors 50 --conf spark.yarn.jars="hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2" - --encoding ${metadataEncoding} - --dateOfCollection ${timestamp} - --provenance ${dataSourceInfo} - --xpath${identifierPath} - --input${sequenceFilePath} - --output${mdStorePath} - -rh${rmq_host} - -ru${rmq_user} - -rp${rmq_pwd} - -rr${rmq_report} - -ro${rmq_ongoing} - -w${workflowId} - - - - - - - - - + + + eu.dnetlib.dhp.actionset.h2020programme.GetFile + --hdfsNameNode${nameNode} + --fileUrl${projectFileURL} + --hdfsPath${workingDir}/projects.csv + + + + + eu.dnetlib.dhp.actionset.h2020programme.GetFile + --hdfsNameNode${nameNode} + --fileUrl${programmeFileURL} + --hdfsPath${workingDir}/programme.csv + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 5a648016ef7061354f4ae795c0ab8c558c52ccff Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 15 May 2020 18:18:50 +0200 Subject: [PATCH 07/88] parameters from the GetFile class --- .../actionset/h2020programme/parameters.json | 91 +++---------------- 1 file changed, 13 insertions(+), 78 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json index 4a6aec5ee..40ad5bf36 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json @@ -1,86 +1,21 @@ [ + { - "paramName": "issm", - "paramLongName": "isSparkSessionManaged", - "paramDescription": "when true will stop SparkSession after job execution", - "paramRequired": false + "paramName": "fu", + "paramLongName" : "fileURL", + "paramDescription" : "the url of the file to download", + "paramRequired" : true }, { - "paramName": "e", - "paramLongName": "encoding", - "paramDescription": "the encoding of the input record should be JSON or XML", - "paramRequired": true + "paramName": "hp", + "paramLongName" : "hdfsPath", + "paramDescription" : "where to save the file", + "paramRequired" : true }, { - "paramName": "d", - "paramLongName": "dateOfCollection", - "paramDescription": "the date when the record has been stored", - "paramRequired": true - }, - { - "paramName": "p", - "paramLongName": "provenance", - "paramDescription": "the infos about the provenance of the collected records", - "paramRequired": true - }, - { - "paramName": "x", - "paramLongName": "xpath", - "paramDescription": "the xpath to identify the record identifier", - "paramRequired": true - }, - { - "paramName": "i", - "paramLongName": "input", - "paramDescription": "the path of the sequencial file to read", - "paramRequired": true - }, - { - "paramName": "o", - "paramLongName": "output", - "paramDescription": "the path of the result DataFrame on HDFS", - "paramRequired": true - }, - { - "paramName": "ru", - "paramLongName": "rabbitUser", - "paramDescription": "the user to connect with RabbitMq for messaging", - "paramRequired": true - }, - { - "paramName": "rp", - "paramLongName": "rabbitPassword", - "paramDescription": "the password to connect with RabbitMq for messaging", - "paramRequired": true - }, - { - "paramName": "rh", - "paramLongName": "rabbitHost", - "paramDescription": "the host of the RabbitMq server", - "paramRequired": true - }, - { - "paramName": "ro", - "paramLongName": "rabbitOngoingQueue", - "paramDescription": "the name of the ongoing queue", - "paramRequired": true - }, - { - "paramName": "rr", - "paramLongName": "rabbitReportQueue", - "paramDescription": "the name of the report queue", - "paramRequired": true - }, - { - "paramName": "w", - "paramLongName": "workflowId", - "paramDescription": "the identifier of the dnet Workflow", - "paramRequired": true - }, - { - "paramName": "t", - "paramLongName": "isTest", - "paramDescription": "the name of the report queue", - "paramRequired": false + "paramName": "hnn", + "paramLongName" : "hdfsNameNode", + "paramDescription" : "the name node", + "paramRequired" : true } ] \ No newline at end of file From abc45f2708e7497cd16d1cc09041737be6d07f55 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 18 May 2020 13:04:06 +0200 Subject: [PATCH 08/88] added dnet-45 HttpConnector and related Classes, produced the POJO for projects and programme --- .../project/CollectorPluginErrorLogList.java | 20 ++ .../project/CollectorServiceException.java | 20 ++ .../project}/GetFile.java | 4 +- .../actionmanager/project/HttpConnector.java | 240 ++++++++++++++++++ .../project/PrepareProjects.java | 139 ++++++++++ .../dhp/actionmanager/project/Programme.java | 52 ++++ .../dhp/actionmanager/project/Project.java | 196 ++++++++++++++ .../project/SparkAtomicActionJob.java | 74 ++++++ .../project/action_set_parameters.json | 26 ++ .../project}/oozie_app/workflow.xml | 4 +- .../project}/parameters.json | 0 .../h2020programme/action_set_parameters.json | 0 12 files changed, 771 insertions(+), 4 deletions(-) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorPluginErrorLogList.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorServiceException.java rename dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/{actionset/h2020programme => actionmanager/project}/GetFile.java (92%) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/HttpConnector.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Programme.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Project.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json rename dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/{actionset/h2020programme => actionmanager/project}/oozie_app/workflow.xml (95%) rename dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/{actionset/h2020programme => actionmanager/project}/parameters.json (100%) delete mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/action_set_parameters.json diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorPluginErrorLogList.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorPluginErrorLogList.java new file mode 100644 index 000000000..bc00e4604 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorPluginErrorLogList.java @@ -0,0 +1,20 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.util.LinkedList; + +public class CollectorPluginErrorLogList extends LinkedList { + + private static final long serialVersionUID = -6925786561303289704L; + + @Override + public String toString() { + String log = new String(); + int index = 0; + for (String errorMessage : this) { + log += String.format("Retry #%s: %s / ", index++, errorMessage); + } + return log; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorServiceException.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorServiceException.java new file mode 100644 index 000000000..a417de50d --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorServiceException.java @@ -0,0 +1,20 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +public class CollectorServiceException extends Exception { + + private static final long serialVersionUID = 7523999812098059764L; + + public CollectorServiceException(String string) { + super(string); + } + + public CollectorServiceException(String string, Throwable exception) { + super(string, exception); + } + + public CollectorServiceException(Throwable exception) { + super(exception); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionset/h2020programme/GetFile.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/GetFile.java similarity index 92% rename from dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionset/h2020programme/GetFile.java rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/GetFile.java index 2fed1a0e3..bbf59a20f 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionset/h2020programme/GetFile.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/GetFile.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.actionset.h2020programme; +package eu.dnetlib.dhp.actionmanager.project; import java.io.*; import java.net.URL; @@ -24,7 +24,7 @@ public class GetFile { .toString( GetFile.class .getResourceAsStream( - "/eu/dnetlib/dhp/actionset/h2020programme/parameters.json"))); + "/eu/dnetlib/dhp/actionmanager/project/parameters.json"))); Configuration conf = new Configuration(); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/HttpConnector.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/HttpConnector.java new file mode 100644 index 000000000..63f67f145 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/HttpConnector.java @@ -0,0 +1,240 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.IOException; +import java.io.InputStream; +import java.net.*; +import java.security.GeneralSecurityException; +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * @author jochen, michele, andrea + */ +public class HttpConnector { + + private static final Log log = LogFactory.getLog(HttpConnector.class); + + private int maxNumberOfRetry = 6; + private int defaultDelay = 120; // seconds + private int readTimeOut = 120; // seconds + + private String responseType = null; + + private String userAgent = "Mozilla/5.0 (compatible; OAI; +http://www.openaire.eu)"; + + public HttpConnector() { + CookieHandler.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL)); + } + + /** + * Given the URL returns the content via HTTP GET + * + * @param requestUrl the URL + * @return the content of the downloaded resource + * @throws CollectorServiceException when retrying more than maxNumberOfRetry times + */ + public String getInputSource(final String requestUrl) throws CollectorServiceException { + return attemptDownlaodAsString(requestUrl, 1, new CollectorPluginErrorLogList()); + } + + /** + * Given the URL returns the content as a stream via HTTP GET + * + * @param requestUrl the URL + * @return the content of the downloaded resource as InputStream + * @throws CollectorServiceException when retrying more than maxNumberOfRetry times + */ + public InputStream getInputSourceAsStream(final String requestUrl) throws CollectorServiceException { + return attemptDownload(requestUrl, 1, new CollectorPluginErrorLogList()); + } + + private String attemptDownlaodAsString(final String requestUrl, final int retryNumber, + final CollectorPluginErrorLogList errorList) + throws CollectorServiceException { + try { + InputStream s = attemptDownload(requestUrl, 1, new CollectorPluginErrorLogList()); + try { + return IOUtils.toString(s); + } catch (IOException e) { + log.error("error while retrieving from http-connection occured: " + requestUrl, e); + Thread.sleep(defaultDelay * 1000); + errorList.add(e.getMessage()); + return attemptDownlaodAsString(requestUrl, retryNumber + 1, errorList); + } finally { + IOUtils.closeQuietly(s); + } + } catch (InterruptedException e) { + throw new CollectorServiceException(e); + } + } + + private InputStream attemptDownload(final String requestUrl, final int retryNumber, + final CollectorPluginErrorLogList errorList) + throws CollectorServiceException { + + if (retryNumber > maxNumberOfRetry) { + throw new CollectorServiceException("Max number of retries exceeded. Cause: \n " + errorList); + } + + log.debug("Downloading " + requestUrl + " - try: " + retryNumber); + try { + InputStream input = null; + + try { + final HttpURLConnection urlConn = (HttpURLConnection) new URL(requestUrl).openConnection(); + urlConn.setInstanceFollowRedirects(false); + urlConn.setReadTimeout(readTimeOut * 1000); + urlConn.addRequestProperty("User-Agent", userAgent); + + if (log.isDebugEnabled()) { + logHeaderFields(urlConn); + } + + int retryAfter = obtainRetryAfter(urlConn.getHeaderFields()); + if (retryAfter > 0 && urlConn.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE) { + log.warn("waiting and repeating request after " + retryAfter + " sec."); + Thread.sleep(retryAfter * 1000); + errorList.add("503 Service Unavailable"); + urlConn.disconnect(); + return attemptDownload(requestUrl, retryNumber + 1, errorList); + } else if ((urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) + || (urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP)) { + final String newUrl = obtainNewLocation(urlConn.getHeaderFields()); + log.debug("The requested url has been moved to " + newUrl); + errorList + .add( + String + .format( + "%s %s. Moved to: %s", urlConn.getResponseCode(), urlConn.getResponseMessage(), + newUrl)); + urlConn.disconnect(); + return attemptDownload(newUrl, retryNumber + 1, errorList); + } else if (urlConn.getResponseCode() != HttpURLConnection.HTTP_OK) { + log + .error( + String + .format("HTTP error: %s %s", urlConn.getResponseCode(), urlConn.getResponseMessage())); + Thread.sleep(defaultDelay * 1000); + errorList.add(String.format("%s %s", urlConn.getResponseCode(), urlConn.getResponseMessage())); + urlConn.disconnect(); + return attemptDownload(requestUrl, retryNumber + 1, errorList); + } else { + input = urlConn.getInputStream(); + responseType = urlConn.getContentType(); + return input; + } + } catch (IOException e) { + log.error("error while retrieving from http-connection occured: " + requestUrl, e); + Thread.sleep(defaultDelay * 1000); + errorList.add(e.getMessage()); + return attemptDownload(requestUrl, retryNumber + 1, errorList); + } + } catch (InterruptedException e) { + throw new CollectorServiceException(e); + } + } + + private void logHeaderFields(final HttpURLConnection urlConn) throws IOException { + log.debug("StatusCode: " + urlConn.getResponseMessage()); + + for (Map.Entry> e : urlConn.getHeaderFields().entrySet()) { + if (e.getKey() != null) { + for (String v : e.getValue()) { + log.debug(" key: " + e.getKey() + " - value: " + v); + } + } + } + } + + private int obtainRetryAfter(final Map> headerMap) { + for (String key : headerMap.keySet()) { + if ((key != null) && key.toLowerCase().equals("retry-after") && (headerMap.get(key).size() > 0) + && NumberUtils.isCreatable(headerMap.get(key).get(0))) { + return Integer + .parseInt(headerMap.get(key).get(0)) + 10; + } + } + return -1; + } + + private String obtainNewLocation(final Map> headerMap) throws CollectorServiceException { + for (String key : headerMap.keySet()) { + if ((key != null) && key.toLowerCase().equals("location") && (headerMap.get(key).size() > 0)) { + return headerMap.get(key).get(0); + } + } + throw new CollectorServiceException("The requested url has been MOVED, but 'location' param is MISSING"); + } + + /** + * register for https scheme; this is a workaround and not intended for the use in trusted environments + */ + public void initTrustManager() { + final X509TrustManager tm = new X509TrustManager() { + + @Override + public void checkClientTrusted(final X509Certificate[] xcs, final String string) { + } + + @Override + public void checkServerTrusted(final X509Certificate[] xcs, final String string) { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + try { + final SSLContext ctx = SSLContext.getInstance("TLS"); + ctx.init(null, new TrustManager[] { + tm + }, null); + HttpsURLConnection.setDefaultSSLSocketFactory(ctx.getSocketFactory()); + } catch (GeneralSecurityException e) { + log.fatal(e); + throw new IllegalStateException(e); + } + } + + public int getMaxNumberOfRetry() { + return maxNumberOfRetry; + } + + public void setMaxNumberOfRetry(final int maxNumberOfRetry) { + this.maxNumberOfRetry = maxNumberOfRetry; + } + + public int getDefaultDelay() { + return defaultDelay; + } + + public void setDefaultDelay(final int defaultDelay) { + this.defaultDelay = defaultDelay; + } + + public int getReadTimeOut() { + return readTimeOut; + } + + public void setReadTimeOut(final int readTimeOut) { + this.readTimeOut = readTimeOut; + } + + public String getResponseType() { + return responseType; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java new file mode 100644 index 000000000..8955edeb4 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java @@ -0,0 +1,139 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.sql.ResultSet; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVFormat; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.common.RelationInverse; +import eu.dnetlib.dhp.schema.oaf.Relation; + +public class PrepareProjects implements Closeable { + private static final Log log = LogFactory.getLog(PrepareProjects.class); + private final Configuration conf; + private final BufferedWriter writer; + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final HttpConnector httpConnector; + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + PrepareProjects.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/parameters.json"))); + + parser.parseArgument(args); + + final String fileURL = parser.get("fileURL"); + final String hdfsPath = parser.get("hdfsPath"); + final String hdfsNameNode = parser.get("hdfsNameNode"); + + try (final PrepareProjects prepareProjects = new PrepareProjects(hdfsPath, hdfsNameNode)) { + + log.info("Getting projects..."); + prepareProjects.execute(fileURL); + + } + } + + public void execute(final String fileURL) throws Exception { + + String projects = httpConnector.getInputSource(fileURL); + final CSVFormat format = CSVFormat.EXCEL + .withHeader() + .withDelimiter(';') + .withQuote('"') + .withTrim(); + final CSVParser parser = CSVParser.parse(projects, format); + final Set headers = parser.getHeaderMap().keySet(); + } + + public List processBlacklistEntry(ResultSet rs) { + try { + Relation direct = new Relation(); + Relation inverse = new Relation(); + + String source_prefix = ModelSupport.entityIdPrefix.get(rs.getString("source_type")); + String target_prefix = ModelSupport.entityIdPrefix.get(rs.getString("target_type")); + + String source_direct = source_prefix + "|" + rs.getString("source"); + direct.setSource(source_direct); + inverse.setTarget(source_direct); + + String target_direct = target_prefix + "|" + rs.getString("target"); + direct.setTarget(target_direct); + inverse.setSource(target_direct); + + String encoding = rs.getString("relationship"); + RelationInverse ri = ModelSupport.relationInverseMap.get(encoding); + direct.setRelClass(ri.getRelation()); + inverse.setRelClass(ri.getInverse()); + direct.setRelType(ri.getRelType()); + inverse.setRelType(ri.getRelType()); + direct.setSubRelType(ri.getSubReltype()); + inverse.setSubRelType(ri.getSubReltype()); + + return Arrays.asList(direct, inverse); + + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws IOException { + writer.close(); + } + + public PrepareProjects( + final String hdfsPath, String hdfsNameNode) + throws Exception { + + this.conf = new Configuration(); + this.conf.set("fs.defaultFS", hdfsNameNode); + this.httpConnector = new HttpConnector(); + FileSystem fileSystem = FileSystem.get(this.conf); + Path hdfsWritePath = new Path(hdfsPath); + FSDataOutputStream fsDataOutputStream = null; + if (fileSystem.exists(hdfsWritePath)) { + fsDataOutputStream = fileSystem.append(hdfsWritePath); + } else { + fsDataOutputStream = fileSystem.create(hdfsWritePath); + } + + this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); + + } + + protected void writeRelation(final Relation r) { + try { + writer.write(OBJECT_MAPPER.writeValueAsString(r)); + writer.newLine(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Programme.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Programme.java new file mode 100644 index 000000000..20877b1a1 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Programme.java @@ -0,0 +1,52 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.Serializable; + +public class Programme implements Serializable { + private String rcn; + private String code; + private String title; + private String shortTitle; + private String language; + + public String getRcn() { + return rcn; + } + + public void setRcn(String rcn) { + this.rcn = rcn; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getShortTitle() { + return shortTitle; + } + + public void setShortTitle(String shortTitle) { + this.shortTitle = shortTitle; + } + + public String getLanguage() { + return language; + } + + public void setLanguage(String language) { + this.language = language; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Project.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Project.java new file mode 100644 index 000000000..abee7f861 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Project.java @@ -0,0 +1,196 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.Serializable; + +public class Project implements Serializable { + private String rcn; + private String id; + private String acronym; + private String status; + private String programme; + private String topics; + private String frameworkProgramme; + private String title; + private String startDate; + private String endDate; + private String projectUrl; + private String objective; + private String totalCost; + private String ecMaxContribution; + private String call; + private String fundingScheme; + private String coordinator; + private String coordinatorCountry; + private String participants; + private String participantCountries; + private String subjects; + + public String getRcn() { + return rcn; + } + + public void setRcn(String rcn) { + this.rcn = rcn; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(String acronym) { + this.acronym = acronym; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getProgramme() { + return programme; + } + + public void setProgramme(String programme) { + this.programme = programme; + } + + public String getTopics() { + return topics; + } + + public void setTopics(String topics) { + this.topics = topics; + } + + public String getFrameworkProgramme() { + return frameworkProgramme; + } + + public void setFrameworkProgramme(String frameworkProgramme) { + this.frameworkProgramme = frameworkProgramme; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } + + public String getProjectUrl() { + return projectUrl; + } + + public void setProjectUrl(String projectUrl) { + this.projectUrl = projectUrl; + } + + public String getObjective() { + return objective; + } + + public void setObjective(String objective) { + this.objective = objective; + } + + public String getTotalCost() { + return totalCost; + } + + public void setTotalCost(String totalCost) { + this.totalCost = totalCost; + } + + public String getEcMaxContribution() { + return ecMaxContribution; + } + + public void setEcMaxContribution(String ecMaxContribution) { + this.ecMaxContribution = ecMaxContribution; + } + + public String getCall() { + return call; + } + + public void setCall(String call) { + this.call = call; + } + + public String getFundingScheme() { + return fundingScheme; + } + + public void setFundingScheme(String fundingScheme) { + this.fundingScheme = fundingScheme; + } + + public String getCoordinator() { + return coordinator; + } + + public void setCoordinator(String coordinator) { + this.coordinator = coordinator; + } + + public String getCoordinatorCountry() { + return coordinatorCountry; + } + + public void setCoordinatorCountry(String coordinatorCountry) { + this.coordinatorCountry = coordinatorCountry; + } + + public String getParticipants() { + return participants; + } + + public void setParticipants(String participants) { + this.participants = participants; + } + + public String getParticipantCountries() { + return participantCountries; + } + + public void setParticipantCountries(String participantCountries) { + this.participantCountries = participantCountries; + } + + public String getSubjects() { + return subjects; + } + + public void setSubjects(String subjects) { + this.subjects = subjects; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java new file mode 100644 index 000000000..b8703378e --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java @@ -0,0 +1,74 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; + +public class SparkAtomicActionJob { + private static final Logger log = LoggerFactory.getLogger(SparkAtomicActionJob.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + SparkAtomicActionJob.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String projectPath = parser.get("projectPath"); + log.info("projectPath: {}", projectPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + final String programmePath = parser.get("programmePath"); + log.info("programmePath {}: ", programmePath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + getAtomicActions( + spark, + projectPath, + programmePath, + outputPath); + }); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + private static void getAtomicActions(SparkSession spark, String projectPatj, String programmePath, + String outputPath) { + + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json new file mode 100644 index 000000000..ca9ae9e97 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false +}, +{ +"paramName": "pjfu", +"paramLongName": "projectsFileURL", +"paramDescription": "the URL from where to get the projects file", +"paramRequired": true +}, +{ +"paramName": "pfu", +"paramLongName": "programmeFileURL", +"paramDescription": "the URL from where to get the programme file", +"paramRequired": true +}, +{ +"paramName": "o", +"paramLongName": "outputPath", +"paramDescription": "the path of the new ActionSet", +"paramRequired": true +} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml similarity index 95% rename from dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index 9b200c2a9..5bfa2e7c4 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -33,7 +33,7 @@ eu.dnetlib.dhp.actionset.h2020programme.GetFile --hdfsNameNode${nameNode} - --fileUrl${projectFileURL} + --fileURL${projectFileURL} --hdfsPath${workingDir}/projects.csv @@ -44,7 +44,7 @@ eu.dnetlib.dhp.actionset.h2020programme.GetFile --hdfsNameNode${nameNode} - --fileUrl${programmeFileURL} + --fileURL${programmeFileURL} --hdfsPath${workingDir}/programme.csv diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json similarity index 100% rename from dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/parameters.json rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/action_set_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/action_set_parameters.json deleted file mode 100644 index e69de29bb..000000000 From 4f1ff7ba73690c64f2f35fca03a959ac605d4247 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 18 May 2020 13:04:39 +0200 Subject: [PATCH 09/88] added dependency to org.apache.commons common-csv --- dhp-workflows/dhp-aggregation/pom.xml | 50 ++++----------------------- 1 file changed, 6 insertions(+), 44 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index 1c5465c14..361517af9 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -38,48 +38,6 @@ ${project.version} - - eu.dnetlib - dnet-actionmanager-common - - - eu.dnetlib - dnet-openaireplus-mapping-utils - - - saxonica - saxon - - - saxonica - saxon-dom - - - jgrapht - jgrapht - - - net.sf.ehcache - ehcache - - - org.springframework - spring-test - - - org.apache.* - * - - - apache - * - - - - - eu.dnetlib - dnet-openaire-data-protos - net.sf.saxon @@ -100,11 +58,15 @@ jaxen + - org.apache.hadoop - hadoop-distcp + org.apache.commons + commons-csv + 1.8 + + From 23bbac7d7cb20f9cadb0d68fcfcaf2dafe094b9b Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 18 May 2020 13:05:03 +0200 Subject: [PATCH 10/88] - --- .../dhp/actionmanager/project/action_set_parameters.json | 8 ++++---- .../dhp/actionmanager/project/oozie_app/workflow.xml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json index ca9ae9e97..a0856e10e 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/action_set_parameters.json @@ -6,14 +6,14 @@ "paramRequired": false }, { -"paramName": "pjfu", -"paramLongName": "projectsFileURL", +"paramName": "pjp", +"paramLongName": "projectPath", "paramDescription": "the URL from where to get the projects file", "paramRequired": true }, { -"paramName": "pfu", -"paramLongName": "programmeFileURL", +"paramName": "pp", +"paramLongName": "programmePath", "paramDescription": "the URL from where to get the programme file", "paramRequired": true }, diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index 5bfa2e7c4..992c2ded1 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -31,7 +31,7 @@ - eu.dnetlib.dhp.actionset.h2020programme.GetFile + eu.dnetlib.dhp.actionmanager.project.GetFile --hdfsNameNode${nameNode} --fileURL${projectFileURL} --hdfsPath${workingDir}/projects.csv @@ -42,7 +42,7 @@ - eu.dnetlib.dhp.actionset.h2020programme.GetFile + eu.dnetlib.dhp.actionmanager.project.GetFile --hdfsNameNode${nameNode} --fileURL${programmeFileURL} --hdfsPath${workingDir}/programme.csv @@ -56,7 +56,7 @@ - + From f0f14caf996828237e7ce6bd4a3720c7ae01ae0f Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 18 May 2020 13:06:16 +0200 Subject: [PATCH 11/88] removed script files for shell actions not performed --- .../h2020programme/oozie_app/lib/scripts/getprogrammefile.sh | 0 .../h2020programme/oozie_app/lib/scripts/getprojectfile.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprogrammefile.sh delete mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprojectfile.sh diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprogrammefile.sh b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprogrammefile.sh deleted file mode 100644 index e69de29bb..000000000 diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprojectfile.sh b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionset/h2020programme/oozie_app/lib/scripts/getprojectfile.sh deleted file mode 100644 index e69de29bb..000000000 From 9447d78ef38abc918e6f0b7bb4f338ebe9f02c86 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 19 May 2020 18:42:50 +0200 Subject: [PATCH 12/88] added preparation classes --- .../dhp/actionmanager/project/GetFile.java | 53 ----- .../project/PrepareProgramme.java | 124 ++++++++++++ .../project/PrepareProjects.java | 189 ++++++++---------- .../project/SparkAtomicActionJob.java | 61 +++++- .../project/csvutils/CSVParser.java | 37 ++++ .../CSVProgramme.java} | 4 +- .../CSVProject.java} | 5 +- .../project/csvutils/ReadCSV.java | 98 +++++++++ .../CollectorPluginErrorLogList.java | 2 +- .../CollectorServiceException.java | 2 +- .../{ => httpconnector}/HttpConnector.java | 2 +- .../project/prepare_programme_parameters.json | 26 +++ .../project/prepare_project_parameters.json | 20 ++ .../actionmanager/project/CSVParserTest.java | 43 ++++ .../project/PrepareProgrammeTest.java | 4 + .../project/SparkUpdateProjectTest.java | 4 + .../httpconnector/HttpConnectorTest.java | 4 + .../preparedProgramme_whole.json | 0 .../dhp/actionmanager/project/programme.csv | 0 .../dhp/actionmanager/projects_subset.json | 0 .../dhp/actionmanager/whole_programme.json | 0 21 files changed, 507 insertions(+), 171 deletions(-) delete mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/GetFile.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVParser.java rename dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/{Programme.java => csvutils/CSVProgramme.java} (87%) rename dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/{Project.java => csvutils/CSVProject.java} (97%) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java rename dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/{ => httpconnector}/CollectorPluginErrorLogList.java (86%) rename dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/{ => httpconnector}/CollectorServiceException.java (86%) rename dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/{ => httpconnector}/HttpConnector.java (99%) create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/GetFile.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/GetFile.java deleted file mode 100644 index bbf59a20f..000000000 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/GetFile.java +++ /dev/null @@ -1,53 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.project; - -import java.io.*; -import java.net.URL; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -import eu.dnetlib.dhp.application.ArgumentApplicationParser; - -public class GetFile { - - private static final Log log = LogFactory.getLog(GetFile.class); - - public static void main(final String[] args) throws Exception { - final ArgumentApplicationParser parser = new ArgumentApplicationParser( - IOUtils - .toString( - GetFile.class - .getResourceAsStream( - "/eu/dnetlib/dhp/actionmanager/project/parameters.json"))); - - Configuration conf = new Configuration(); - - parser.parseArgument(args); - - final String fileURL = parser.get("fileURL"); - final String hdfsPath = parser.get("hdfsPath"); - final String hdfsNameNode = parser.get("hdfsNameNode"); - - conf.set("fs.defaultFS", hdfsNameNode); - FileSystem fileSystem = FileSystem.get(conf); - Path hdfsWritePath = new Path(hdfsPath); - FSDataOutputStream fsDataOutputStream = null; - if (fileSystem.exists(hdfsWritePath)) { - fsDataOutputStream = fileSystem.append(hdfsWritePath); - } else { - fsDataOutputStream = fileSystem.create(hdfsWritePath); - } - - InputStream is = new BufferedInputStream(new URL(fileURL).openStream()); - - org.apache.hadoop.io.IOUtils.copyBytes(is, fsDataOutputStream, 4096, true); - - } - -} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java new file mode 100644 index 000000000..a5abb9ea7 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java @@ -0,0 +1,124 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.HashMap; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import scala.Tuple2; + +public class PrepareProgramme { + + private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final HashMap programmeMap = new HashMap<>(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + PrepareProgramme.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String programmePath = parser.get("programmePath"); + log.info("programmePath {}: ", programmePath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + exec(spark, programmePath, outputPath); + }); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + private static void exec(SparkSession spark, String programmePath, String outputPath) { + Dataset programme = readPath(spark, programmePath, CSVProgramme.class); + + programme + .toJavaRDD() + .filter(p -> !p.getCode().contains("FP7")) + .mapToPair(csvProgramme -> new Tuple2<>(csvProgramme.getCode(), csvProgramme)) + .reduceByKey((a, b) -> { + if (StringUtils.isEmpty(a.getShortTitle())) { + if (StringUtils.isEmpty(b.getShortTitle())) { + if (StringUtils.isEmpty(a.getTitle())) { + if (StringUtils.isNotEmpty(b.getTitle())) { + a.setShortTitle(b.getTitle()); + a.setLanguage(b.getLanguage()); + } + } else {// notIsEmpty a.getTitle + if (StringUtils.isEmpty(b.getTitle())) { + a.setShortTitle(a.getTitle()); + } else { + if (b.getLanguage().equalsIgnoreCase("en")) { + a.setShortTitle(b.getTitle()); + a.setLanguage(b.getLanguage()); + } else { + a.setShortTitle(a.getTitle()); + } + } + } + } else {// not isEmpty b.getShortTitle + a.setShortTitle(b.getShortTitle()); + // a.setLanguage(b.getLanguage()); + } + } + return a; + + }) + .map(p -> { + CSVProgramme csvProgramme = p._2(); + if (StringUtils.isEmpty(csvProgramme.getShortTitle())) { + csvProgramme.setShortTitle(csvProgramme.getTitle()); + } + return OBJECT_MAPPER.writeValueAsString(csvProgramme); + }) + .saveAsTextFile(outputPath); + + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java index 8955edeb4..1c98199f8 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java @@ -1,139 +1,108 @@ package eu.dnetlib.dhp.actionmanager.project; -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.sql.ResultSet; -import java.util.Arrays; -import java.util.List; -import java.util.Set; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.schema.common.ModelSupport; -import eu.dnetlib.dhp.schema.common.RelationInverse; -import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.dhp.common.HdfsSupport; +import scala.Tuple2; -public class PrepareProjects implements Closeable { - private static final Log log = LogFactory.getLog(PrepareProjects.class); - private final Configuration conf; - private final BufferedWriter writer; - private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final HttpConnector httpConnector; +public class PrepareProjects { - public static void main(final String[] args) throws Exception { - final ArgumentApplicationParser parser = new ArgumentApplicationParser( - IOUtils - .toString( - PrepareProjects.class - .getResourceAsStream( - "/eu/dnetlib/dhp/actionmanager/project/parameters.json"))); + private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final HashMap programmeMap = new HashMap<>(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + PrepareProjects.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); parser.parseArgument(args); - final String fileURL = parser.get("fileURL"); - final String hdfsPath = parser.get("hdfsPath"); - final String hdfsNameNode = parser.get("hdfsNameNode"); + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); - try (final PrepareProjects prepareProjects = new PrepareProjects(hdfsPath, hdfsNameNode)) { + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - log.info("Getting projects..."); - prepareProjects.execute(fileURL); + final String projectPath = parser.get("projectPath"); + log.info("projectPath {}: ", projectPath); - } + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + exec(spark, projectPath, outputPath); + }); } - public void execute(final String fileURL) throws Exception { - - String projects = httpConnector.getInputSource(fileURL); - final CSVFormat format = CSVFormat.EXCEL - .withHeader() - .withDelimiter(';') - .withQuote('"') - .withTrim(); - final CSVParser parser = CSVParser.parse(projects, format); - final Set headers = parser.getHeaderMap().keySet(); + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } - public List processBlacklistEntry(ResultSet rs) { - try { - Relation direct = new Relation(); - Relation inverse = new Relation(); + private static void exec(SparkSession spark, String progjectPath, String outputPath) { + Dataset project = readPath(spark, progjectPath, CSVProject.class); - String source_prefix = ModelSupport.entityIdPrefix.get(rs.getString("source_type")); - String target_prefix = ModelSupport.entityIdPrefix.get(rs.getString("target_type")); + project + .toJavaRDD() + .flatMap(p -> { + List csvProjectList = new ArrayList<>(); + String[] programme = p.getProgramme().split(";"); + if (programme.length > 1) { + for (int i = 0; i < programme.length; i++) { + CSVProject csvProject = new CSVProject(); + csvProject.setProgramme(programme[i]); + csvProjectList.add(csvProject); + } + } else { + csvProjectList.add(p); + } - String source_direct = source_prefix + "|" + rs.getString("source"); - direct.setSource(source_direct); - inverse.setTarget(source_direct); - - String target_direct = target_prefix + "|" + rs.getString("target"); - direct.setTarget(target_direct); - inverse.setSource(target_direct); - - String encoding = rs.getString("relationship"); - RelationInverse ri = ModelSupport.relationInverseMap.get(encoding); - direct.setRelClass(ri.getRelation()); - inverse.setRelClass(ri.getInverse()); - direct.setRelType(ri.getRelType()); - inverse.setRelType(ri.getRelType()); - direct.setSubRelType(ri.getSubReltype()); - inverse.setSubRelType(ri.getSubReltype()); - - return Arrays.asList(direct, inverse); - - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void close() throws IOException { - writer.close(); - } - - public PrepareProjects( - final String hdfsPath, String hdfsNameNode) - throws Exception { - - this.conf = new Configuration(); - this.conf.set("fs.defaultFS", hdfsNameNode); - this.httpConnector = new HttpConnector(); - FileSystem fileSystem = FileSystem.get(this.conf); - Path hdfsWritePath = new Path(hdfsPath); - FSDataOutputStream fsDataOutputStream = null; - if (fileSystem.exists(hdfsWritePath)) { - fsDataOutputStream = fileSystem.append(hdfsWritePath); - } else { - fsDataOutputStream = fileSystem.create(hdfsWritePath); - } - - this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); + return csvProjectList.iterator(); + }) + .map(p -> OBJECT_MAPPER.writeValueAsString(p)) + .saveAsTextFile(outputPath); } - protected void writeRelation(final Relation r) { - try { - writer.write(OBJECT_MAPPER.writeValueAsString(r)); - writer.newLine(); - } catch (final Exception e) { - throw new RuntimeException(e); - } + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); } - } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java index b8703378e..61bd952db 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java @@ -3,22 +3,35 @@ package eu.dnetlib.dhp.actionmanager.project; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; +import java.util.Arrays; +import java.util.HashMap; import java.util.Optional; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.Programme; +import eu.dnetlib.dhp.schema.oaf.Project; +import eu.dnetlib.dhp.utils.DHPUtils; public class SparkAtomicActionJob { private static final Logger log = LoggerFactory.getLogger(SparkAtomicActionJob.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final HashMap programmeMap = new HashMap<>(); public static void main(String[] args) throws Exception { @@ -67,8 +80,54 @@ public class SparkAtomicActionJob { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } - private static void getAtomicActions(SparkSession spark, String projectPatj, String programmePath, + private static void getAtomicActions(SparkSession spark, String projectPatH, + String programmePath, String outputPath) { + Dataset project = readPath(spark, projectPatH, CSVProject.class); + Dataset programme = readPath(spark, programmePath, CSVProgramme.class); + + project + .joinWith(programme, project.col("programme").equalTo(programme.col("code")), "left") + .map(c -> { + CSVProject csvProject = c._1(); + Optional csvProgramme = Optional.ofNullable(c._2()); + if (csvProgramme.isPresent()) { + Project p = new Project(); + p + .setId( + createOpenaireId( + ModelSupport.entityIdPrefix.get("project"), + "corda__h2020", csvProject.getId())); + Programme pm = new Programme(); + pm.setCode(csvProject.getProgramme()); + pm.setDescription(csvProgramme.get().getShortTitle()); + p.setProgramme(Arrays.asList(pm)); + return p; + } + + return null; + }, Encoders.bean(Project.class)) + .filter(p -> !(p == null)) + // .map(p -> new AtomicAction<>(Project.class, p), Encoders.bean(AtomicAction.class)) + .write() + .option("compression", "gzip") + .mode(SaveMode.Overwrite) + .json(outputPath); + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + + public static String createOpenaireId( + final String prefix, final String nsPrefix, final String id) { + + return String.format("%s|%s::%s", prefix, nsPrefix, DHPUtils.md5(id)); + } } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVParser.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVParser.java new file mode 100644 index 000000000..ef29a6b6a --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVParser.java @@ -0,0 +1,37 @@ + +package eu.dnetlib.dhp.actionmanager.project.csvutils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang.reflect.FieldUtils; + +public class CSVParser { + + public List parse(String csvFile, String classForName) + throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException { + final CSVFormat format = CSVFormat.EXCEL + .withHeader() + .withDelimiter(';') + .withQuote('"') + .withTrim(); + List ret = new ArrayList<>(); + final org.apache.commons.csv.CSVParser parser = org.apache.commons.csv.CSVParser.parse(csvFile, format); + final Set headers = parser.getHeaderMap().keySet(); + Class clazz = Class.forName(classForName); + for (CSVRecord csvRecord : parser.getRecords()) { + final Object cc = clazz.newInstance(); + for (String header : headers) { + FieldUtils.writeField(cc, header, csvRecord.get(header), true); + + } + ret.add((R) cc); + } + + return ret; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Programme.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProgramme.java similarity index 87% rename from dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Programme.java rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProgramme.java index 20877b1a1..a9069e510 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Programme.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProgramme.java @@ -1,9 +1,9 @@ -package eu.dnetlib.dhp.actionmanager.project; +package eu.dnetlib.dhp.actionmanager.project.csvutils; import java.io.Serializable; -public class Programme implements Serializable { +public class CSVProgramme implements Serializable { private String rcn; private String code; private String title; diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Project.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProject.java similarity index 97% rename from dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Project.java rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProject.java index abee7f861..ff18c6260 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/Project.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/CSVProject.java @@ -1,9 +1,9 @@ -package eu.dnetlib.dhp.actionmanager.project; +package eu.dnetlib.dhp.actionmanager.project.csvutils; import java.io.Serializable; -public class Project implements Serializable { +public class CSVProject implements Serializable { private String rcn; private String id; private String acronym; @@ -193,4 +193,5 @@ public class Project implements Serializable { public void setSubjects(String subjects) { this.subjects = subjects; } + } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java new file mode 100644 index 000000000..905194232 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java @@ -0,0 +1,98 @@ + +package eu.dnetlib.dhp.actionmanager.project.csvutils; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.httpconnector.HttpConnector; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class ReadCSV implements Closeable { + private static final Log log = LogFactory.getLog(ReadCSV.class); + private final Configuration conf; + private final BufferedWriter writer; + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private String csvFile; + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + ReadCSV.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/parameters.json"))); + + parser.parseArgument(args); + + final String fileURL = parser.get("fileURL"); + final String hdfsPath = parser.get("hdfsPath"); + final String hdfsNameNode = parser.get("hdfsNameNode"); + final String classForName = parser.get("classForName"); + + try (final ReadCSV readCSV = new ReadCSV(hdfsPath, hdfsNameNode, fileURL)) { + + log.info("Getting CSV file..."); + readCSV.execute(classForName); + + } + } + + public void execute(final String classForName) throws Exception { + CSVParser csvParser = new CSVParser(); + csvParser + .parse(csvFile, classForName) + .stream() + .forEach(p -> write(p)); + + } + + @Override + public void close() throws IOException { + writer.close(); + } + + public ReadCSV( + final String hdfsPath, + final String hdfsNameNode, + final String fileURL) + throws Exception { + this.conf = new Configuration(); + this.conf.set("fs.defaultFS", hdfsNameNode); + HttpConnector httpConnector = new HttpConnector(); + FileSystem fileSystem = FileSystem.get(this.conf); + Path hdfsWritePath = new Path(hdfsPath); + FSDataOutputStream fsDataOutputStream = null; + if (fileSystem.exists(hdfsWritePath)) { + fsDataOutputStream = fileSystem.append(hdfsWritePath); + } else { + fsDataOutputStream = fileSystem.create(hdfsWritePath); + } + + this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); + this.csvFile = httpConnector.getInputSource(fileURL); + ; + } + + protected void write(final Object p) { + try { + writer.write(OBJECT_MAPPER.writeValueAsString(p)); + writer.newLine(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorPluginErrorLogList.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorPluginErrorLogList.java similarity index 86% rename from dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorPluginErrorLogList.java rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorPluginErrorLogList.java index bc00e4604..9d3f88265 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorPluginErrorLogList.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorPluginErrorLogList.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.actionmanager.project; +package eu.dnetlib.dhp.actionmanager.project.httpconnector; import java.util.LinkedList; diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorServiceException.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorServiceException.java similarity index 86% rename from dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorServiceException.java rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorServiceException.java index a417de50d..9167d97b4 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/CollectorServiceException.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/CollectorServiceException.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.actionmanager.project; +package eu.dnetlib.dhp.actionmanager.project.httpconnector; public class CollectorServiceException extends Exception { diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/HttpConnector.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnector.java similarity index 99% rename from dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/HttpConnector.java rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnector.java index 63f67f145..e20518b55 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/HttpConnector.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnector.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.actionmanager.project; +package eu.dnetlib.dhp.actionmanager.project.httpconnector; import java.io.IOException; import java.io.InputStream; diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json new file mode 100644 index 000000000..a0856e10e --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false +}, +{ +"paramName": "pjp", +"paramLongName": "projectPath", +"paramDescription": "the URL from where to get the projects file", +"paramRequired": true +}, +{ +"paramName": "pp", +"paramLongName": "programmePath", +"paramDescription": "the URL from where to get the programme file", +"paramRequired": true +}, +{ +"paramName": "o", +"paramLongName": "outputPath", +"paramDescription": "the path of the new ActionSet", +"paramRequired": true +} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json new file mode 100644 index 000000000..54083e108 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json @@ -0,0 +1,20 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false +}, +{ +"paramName": "pp", +"paramLongName": "programmePath", +"paramDescription": "the URL from where to get the programme file", +"paramRequired": true +}, +{ +"paramName": "o", +"paramLongName": "outputPath", +"paramDescription": "the path of the new ActionSet", +"paramRequired": true +} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java new file mode 100644 index 000000000..d344f3118 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java @@ -0,0 +1,43 @@ +package eu.dnetlib.dhp.actionmanager.project; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class ReadCSVTest { + + private static Path workingDir; + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(eu.dnetlib.dhp.actionmanager.project.ReadCSVTest.class.getSimpleName()); + + + } + @Test + public void readProgrammeTest() throws Exception { + + String programmecsv = IOUtils.toString(getClass() + .getClassLoader().getResourceAsStream("eu/dnetlib/dhp/actionmanager/project/programme.csv")); + ReadCSV + .main( + new String[] { + "-fileURL", + "http://cordis.europa.eu/data/reference/cordisref-H2020programmes.csv", + "-outputPath", + workingDir.toString() + "/project", + "-hdfsPath", + getClass().getResource("/eu/dnetlib/dhp/blacklist/blacklist").getPath(), + "-mergesPath", + getClass().getResource("/eu/dnetlib/dhp/blacklist/mergesRelOneMerge").getPath(), + }); + + + + + } +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java new file mode 100644 index 000000000..b22e6bd6d --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java @@ -0,0 +1,4 @@ +package eu.dnetlib.dhp.actionmanager.project; + +public class PrepareProgrammeTest { +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java new file mode 100644 index 000000000..f7d271fe0 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -0,0 +1,4 @@ +package eu.dnetlib.dhp.actionmanager.project; + +public class SparkUpdateProjectSet { +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java new file mode 100644 index 000000000..29e9a6cce --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java @@ -0,0 +1,4 @@ +package eu.dnetlib.dhp.actionmanager.project.httpconnector; + +public class HttpConnectorTest { +} diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json new file mode 100644 index 000000000..e69de29bb From 457293ccc0c86f1cb5e51b179772302102b18c81 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 19 May 2020 18:43:42 +0200 Subject: [PATCH 13/88] test for the variuos steps of project update with programme --- .../actionmanager/project/CSVParserTest.java | 50 +++++----- .../project/PrepareProgrammeTest.java | 90 ++++++++++++++++++ .../project/SparkUpdateProjectTest.java | 92 ++++++++++++++++++- .../httpconnector/HttpConnectorTest.java | 35 +++++++ 4 files changed, 240 insertions(+), 27 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java index d344f3118..17fdd4511 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java @@ -1,43 +1,41 @@ + package eu.dnetlib.dhp.actionmanager.project; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVParser; -public class ReadCSVTest { +public class CSVParserTest { - private static Path workingDir; + private static Path workingDir; - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files.createTempDirectory(eu.dnetlib.dhp.actionmanager.project.ReadCSVTest.class.getSimpleName()); + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(CSVParserTest.class.getSimpleName()); + } - } - @Test - public void readProgrammeTest() throws Exception { + @Test + public void readProgrammeTest() throws Exception { - String programmecsv = IOUtils.toString(getClass() - .getClassLoader().getResourceAsStream("eu/dnetlib/dhp/actionmanager/project/programme.csv")); - ReadCSV - .main( - new String[] { - "-fileURL", - "http://cordis.europa.eu/data/reference/cordisref-H2020programmes.csv", - "-outputPath", - workingDir.toString() + "/project", - "-hdfsPath", - getClass().getResource("/eu/dnetlib/dhp/blacklist/blacklist").getPath(), - "-mergesPath", - getClass().getResource("/eu/dnetlib/dhp/blacklist/mergesRelOneMerge").getPath(), - }); + String programmecsv = IOUtils + .toString( + getClass() + .getClassLoader() + .getResourceAsStream("eu/dnetlib/dhp/actionmanager/project/programme.csv")); + CSVParser csvParser = new CSVParser(); + List pl = csvParser.parse(programmecsv, "eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme"); + System.out.println(pl.size()); - } + } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java index b22e6bd6d..50804f75e 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java @@ -1,4 +1,94 @@ + package eu.dnetlib.dhp.actionmanager.project; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; + public class PrepareProgrammeTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final ClassLoader cl = eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class + .getClassLoader(); + + private static SparkSession spark; + + private static Path workingDir; + private static final Logger log = LoggerFactory + .getLogger(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(eu.dnetlib.dhp.actionmanager.project.PrepareProgrammeTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(PrepareProgrammeTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + public void numberDistinctProgrammeTest() throws Exception { + PrepareProgramme + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-programmePath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/whole_programme.json").getPath(), + "-outputPath", + workingDir.toString() + "/preparedProgramme" + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/preparedProgramme") + .map(item -> OBJECT_MAPPER.readValue(item, CSVProgramme.class)); + + Assertions.assertEquals(277, tmp.count()); + + Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProgramme.class)); + + Assertions.assertEquals(0, verificationDataset.filter("shortTitle =''").count()); + } + } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index f7d271fe0..d48884842 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -1,4 +1,94 @@ + package eu.dnetlib.dhp.actionmanager.project; -public class SparkUpdateProjectSet { +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.schema.oaf.Project; + +public class SparkUpdateProjectTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final ClassLoader cl = eu.dnetlib.dhp.actionmanager.project.SparkUpdateProjectTest.class + .getClassLoader(); + + private static SparkSession spark; + + private static Path workingDir; + private static final Logger log = LoggerFactory + .getLogger(eu.dnetlib.dhp.actionmanager.project.SparkUpdateProjectTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(eu.dnetlib.dhp.actionmanager.project.SparkUpdateProjectTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(eu.dnetlib.dhp.actionmanager.project.SparkUpdateProjectTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(SparkUpdateProjectTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + public void numberDistinctProgrammeTest() throws Exception { + SparkAtomicActionJob + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-programmePath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json").getPath(), + "-projectPath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/projects_subset.json").getPath(), + "-outputPath", + workingDir.toString() + "/actionSet" + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/actionSet") + .map(item -> OBJECT_MAPPER.readValue(item, Project.class)); + + Assertions.assertEquals(14, tmp.count()); + +// Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProgramme.class)); +// +// Assertions.assertEquals(0, verificationDataset.filter("shortTitle =''").count()); + } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java index 29e9a6cce..51a7019ca 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/httpconnector/HttpConnectorTest.java @@ -1,4 +1,39 @@ + package eu.dnetlib.dhp.actionmanager.project.httpconnector; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.ssl.SSLContextBuilder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + public class HttpConnectorTest { + + private static final Log log = LogFactory.getLog(HttpConnectorTest.class); + private static HttpConnector connector; + + private static final String URL = "http://cordis.europa.eu/data/reference/cordisref-H2020programmes.csv"; + private static final String URL_MISCONFIGURED_SERVER = "https://www.alexandria.unisg.ch/cgi/oai2?verb=Identify"; + private static final String URL_GOODSNI_SERVER = "https://air.unimi.it/oai/openaire?verb=Identify"; + + private static final SSLContextBuilder sslContextBuilder = new SSLContextBuilder(); + private static SSLConnectionSocketFactory sslSocketFactory; + + @BeforeAll + public static void setUp() { + connector = new HttpConnector(); + } + + @Test + + public void testGetInputSource() throws CollectorServiceException { + System.out.println(connector.getInputSource(URL)); + } + + @Test + public void testGoodServers() throws CollectorServiceException { + System.out.println(connector.getInputSource(URL_GOODSNI_SERVER)); + } + } From 08218d2f3f9fd64423907ebf31508b8213745249 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 19 May 2020 18:44:25 +0200 Subject: [PATCH 14/88] new workflow with added steps --- .../project/oozie_app/workflow.xml | 114 +++++++++++++----- 1 file changed, 82 insertions(+), 32 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index 992c2ded1..cd4d79ab7 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -24,6 +24,7 @@ + @@ -31,51 +32,100 @@ - eu.dnetlib.dhp.actionmanager.project.GetFile + eu.dnetlib.dhp.actionmanager.project.csvutils.ReadCSV --hdfsNameNode${nameNode} --fileURL${projectFileURL} - --hdfsPath${workingDir}/projects.csv + --hdfsPath${workingDir}/projects + --classForNameeu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject - + - + - eu.dnetlib.dhp.actionmanager.project.GetFile + eu.dnetlib.dhp.actionmanager.project.csvutils.ReadCSV --hdfsNameNode${nameNode} --fileURL${programmeFileURL} - --hdfsPath${workingDir}/programme.csv + --hdfsPath${workingDir}/programme + --classForNameeu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme + + + + + + + yarn + cluster + PrepareProgramme + eu.dnetlib.dhp.actionmanager.project.PrepareProgramme + dhp-aggregation-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --programmePath${workingDir}/programme + --outputPath${workingDir}/preparedProgramme + + + + + + + + yarn + cluster + PrepareProgramme + eu.dnetlib.dhp.actionmanager.project.PrepareProjects + dhp-aggregation-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --projectPath${workingDir}/projects + --outputPath${workingDir}/preparedProjects + + + + + + + + yarn + cluster + ProjectProgrammeAS + eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob + dhp-aggregation-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=3840 + + --projectPath${workingDir}/preparedProjects + --programmePath${workingDir}/preparedProgramme + --outputPath/tmp/h2020programme + - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From eb0e47ba53d22e130740215d20a1636144c9071a Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 10:26:44 +0200 Subject: [PATCH 15/88] parameters for h2020 programme --- .../project/oozie_app/config-default.xml | 54 +++++++++++++++++++ .../dhp/actionmanager/project/parameters.json | 10 +++- .../project/prepare_programme_parameters.json | 6 --- .../project/prepare_project_parameters.json | 4 +- 4 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/config-default.xml diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/config-default.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/config-default.xml new file mode 100644 index 000000000..fe82ae194 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/config-default.xml @@ -0,0 +1,54 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + + + sparkExecutorNumber + 4 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + sparkDriverMemory + 15G + + + sparkExecutorMemory + 6G + + + sparkExecutorCores + 1 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json index 40ad5bf36..dd3de70f6 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/parameters.json @@ -17,5 +17,13 @@ "paramLongName" : "hdfsNameNode", "paramDescription" : "the name node", "paramRequired" : true - } + }, + { + "paramName": "cfn", + "paramLongName" : "classForName", + "paramDescription" : "the name of the class to deserialize the csv to", + "paramRequired" : true +} + + ] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json index a0856e10e..54083e108 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_programme_parameters.json @@ -6,12 +6,6 @@ "paramRequired": false }, { -"paramName": "pjp", -"paramLongName": "projectPath", -"paramDescription": "the URL from where to get the projects file", -"paramRequired": true -}, -{ "paramName": "pp", "paramLongName": "programmePath", "paramDescription": "the URL from where to get the programme file", diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json index 54083e108..5fc88ce8e 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json @@ -6,8 +6,8 @@ "paramRequired": false }, { -"paramName": "pp", -"paramLongName": "programmePath", +"paramName": "pjp", +"paramLongName": "projectPath", "paramDescription": "the URL from where to get the programme file", "paramRequired": true }, From d323100af09a8301694a6a712753547dcaecd127 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 10:27:27 +0200 Subject: [PATCH 16/88] added the new Programme POJO. It contains the code and the description of the programme --- .../eu/dnetlib/dhp/schema/oaf/Programme.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java index f91333616..69223ab01 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java @@ -1,4 +1,23 @@ + package eu.dnetlib.dhp.schema.oaf; public class Programme { + private String code; + private String description; + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } } From 24daa1deaaa1b78a14556c4fb53570a5db478aa4 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 10:28:16 +0200 Subject: [PATCH 17/88] added to the Project class a new field that is the list of programmes --- .../main/java/eu/dnetlib/dhp/schema/oaf/Project.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java index 924c08cc9..1eae3e8ee 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java @@ -58,6 +58,8 @@ public class Project extends OafEntity implements Serializable { private Float fundedamount; + private List programme; + public Field getWebsiteurl() { return websiteurl; } @@ -266,6 +268,14 @@ public class Project extends OafEntity implements Serializable { this.fundedamount = fundedamount; } + public List getProgramme() { + return programme; + } + + public void setProgramme(List programme) { + this.programme = programme; + } + @Override public void mergeFrom(OafEntity e) { super.mergeFrom(e); From 75491482de353bbc9f5d7a415028061267d2f43e Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 10:28:56 +0200 Subject: [PATCH 18/88] added a new preparation step to replicate each project for the programme it is associated to --- .../eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java index 1c98199f8..df37f9286 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java @@ -82,9 +82,11 @@ public class PrepareProjects { List csvProjectList = new ArrayList<>(); String[] programme = p.getProgramme().split(";"); if (programme.length > 1) { + String id = p.getId(); for (int i = 0; i < programme.length; i++) { CSVProject csvProject = new CSVProject(); csvProject.setProgramme(programme[i]); + csvProject.setId(id); csvProjectList.add(csvProject); } } else { From faed7521bf100bca898ada798dc76abe96f02e80 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 10:29:29 +0200 Subject: [PATCH 19/88] added resources for testing --- .../preparedProgramme_whole.json.gz | Bin 0 -> 14212 bytes .../dhp/actionmanager/whole_programme.json.gz | Bin 0 -> 34620 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json.gz create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json.gz diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..1afa730616472d8f4e3f76f078ce753bc9cd6b24 GIT binary patch literal 14212 zcmV-~H+#q*iwFqr0K{GZ18{O>aA9&~WKeQ%XL4a}ZDn6~Xm4y~E^2dcZUF6lOK%%l zmgatbMIbkJ0hG%WCEMv8Em1ZrWl1B-?XE-ugJ8rZd97dsGajPJMguj8Dhy^ltLd&8 zbOE&(V^E7h0==nPwbFm2f61J4?v2NN%1SF;nbo<-Dk~U|8#nGf&+mNaA3n>Y^s}d* zJ$kgi*MIm~@3Sb28T|cX_u=ltozbfYg9m#L_TfMHspMsXzl|>rgCWmndAMY7xL9O) zd13!83DbEM&hZzPe)f<5=^vWM?d?6hsHh$1E)9$0Mm}jE5l;JMo{50WrgWy$rQN*;v)tMY8c!XRd_9AGQ3qRgwP ztaA9&bPQgzxFVNFcna^E!XH^0Ci(=Pvx0?rv=Fzh@67P=JlfDZ z%IizIzP`2PoeS-qcWsNSuWtU)-ye*Qk4Gm5qhN9{9-Ul_kH-J`)hO5r#>#l;O>-74 z;2SIqG10t)c9A;3S*{Flf3P=jto}#p>OU9Xzg)nyUVv{=DtgK)KiNu{YPdck_Z6a0En4ttii@q%&q^nAuj5D$2l zgk9nCg)`yZq7a|CT*2yv#aqIG48D9SKT8R7lV(Xa=gbf~Kf&?4_tO4gU|UzaB(ff> z`{`wslzb=VMZ{s`X_#;D9r7%MHB6ccd((xRG$xa?>uGFYr(f>Htm&}=6MX&k1dyd+$e;S}ah zd3dl0ukg|$TP^4@Vaep)gZoN>md$329)J^>u($B>(n!o8FJb0_X^3AfOMCQo>}`mr z_2|>X7FmK|Ea;wdv4ipZ7c9lk*9&m;a1{Rua|DkRhZ#uD`34VN@dB^GfuyWlXF1AV zLn8k?=bvKEeWuW3mnrl}5xr?afS|FlxAyzCWu|<&<9p-D z#rWdY#c42lb$)s_It>oT!T99l^zVjWj!#cU!6>*G9h{sVpFSTCgZrcL#V?GFD2&k* zHW|EqS%ET#X&$F(b|s`64*#N7=9}gwossiWRp@&UpEx~5WzNUL!?WNB?7QGRD>Gs7 z8y=(m9M6i}-M26MKK<8u`~?UD6x9JJHxLV~D|w{e!hhy$j^9mu(j|*|#8bm!{m6Uy z#LW|zRjaMwVXa0xHlfr(mSuXw$v$ zqhJz%C4v@9^Y9oZJ}*G+oD{DwFwh#-k8gI zjElFxMJiORVZ`fFcHb&xQ4QcIlx6WOLq~^K2~N z?|BcO9(!b)#9bP^vds>!-_dpkrx`u-7+j;2OgfyY>vv#{Ff?w1G)yy}6E~5zOXm(C zc$KTP@2>f~dwUMeX3Y3S4w?;3+I$YWx2PSy6?}ZKkr~?TEWhHlflQ$SG=$4VSfa@U zmMRJcT$|`hRy@IVUGRj1UX+p!7Yp1Hwg6na3dQDtWh@j(MB=f#oY1xt3mtDDFRYR< z4@waNqJ0=fv;l)9m&SHavAprtF>;%tQ0nW>UaadgP^UlEi zUVy-P4dY|v+XSz(MFQ3&TQDPt`U2eMa-ohF#-)z;bGz`^)gyWn*gPTl{pf7NRAI2v zP@ax+T+{XV;luu}%SL>nPFcqiJNf`6U<`vzhsUUp8J)C%z(b z2+Do@OJL6$&>c`dXIEVU6wzt)?G< zf0oCvyFn(^2@G`ozpgMMCCo@zlVHWb%%vE80z-jPNIVVG+^Z0U+XVrWWOu1^qo36`8+{`=wwwb-&SGzUs&zJgZ02D z(&7k!-{}Lqa0-`@>{{HTi_=MPI0}x3!Q}K{JUSd6f;~!}D(p~rdco$DrvCTe{0|Hl zfh!`s;Dk?EUcx4(Y&D%#?`aa1A^cEOMG)}{4ug+t3MCB4JHrh|F52ouXz}->>)*dP zADzJaoSz;akKo15gYn7XtI5Ur7&FpG*MC`nB!n@ZX9*a1Q^tdN4%S$LQMwqGctzRr z7_8fzgFgUAE?S4cu*!~Eh$)8!U!jggtTh*YD;zEb;4q0rd{+7n1`iNWZXpiwphh3XvKxfVje-KCD~tAmOL8 zmm~Q8jaSd%4l7i5_s9%dUcUnyhr?=*?^P&AaeI>mU#za*MSRBLzNX1f=do~!YYLQG zooDsOi1v3q5^rGpCa6+%ucD~^J0Yh!(=7o2DFsi;;8lvVK`;T=r@O=Z`#=TzcHgG2 z1IUJ8fG&BIXH(pqXwG5I(3IDn^L)eDb!EqJAc{?NR!84^U;~$nEQWuSStb)mt0XMZ zt%HL(o3WfMPLV?wC!n(6AX`XFh|G17&B`^#ui*DzW;q3!^DNoHMVL}>mo29(j#>Q6 zXan+lXSLWAlsj#5`C%OAB8T0kzTq}08TYlyqD4)cQGCB%C!q*fL#h*Arh} z=3!~TpiyS{G`f)DX5cNr&Hz_qvtk-tMgERMVQ~||(?8qKvwBCf>a#8St~kQLSmH~t z8AR&76EZ?cmZ;`WxNcLu-CP<-KXMac{zMWLrUmW_!Wh36y?WS#H@FfI5B7GIWOeA2 zy)C5+s)93ewo_roE@#muYE(A-g?BOR_n+!Z_|)Medq%GhpRjKaYb-FrDDzisFd7_t zgks9~Ur-n_h{MvbI6me^?x^D6^C$M$@*~|eY+Ydbj_iuZwD%GEBQ=uHR=9x#1f_Az zgbu=!M(0DfZ~gH%_H8+fTCYQQDq;@2qDly>AVfJ>;HsIU_yIWxax4Zx2UahJV_>bM z+0t0>-|rUQ!P~zQ4qgd8A7vOpX|_pF2WF#9;~mF*PB9CIz4*QE;5Bj0p$l`W>pM0_ zbfkqm?8}byT``G$RhD)!LJB-ch7}sw(!6MxHpwFp?KOr^ALSk1*Y}zZ^K78T>`qt( z4rG4G(zHxDtUdC`QatKsz>1~LT#-<8MFgW~;%_JEug!8#gUVE!PaAg8hFv>$!( zKWiFBBw_l7?yRmdMC2kykt@k%bhK9)eg+EGVj>PqmH-tM{9k-eiPb1$J*Y~I?d;H0 z>30QBpKhcCrYQWA``wPgo?W91)D1oiPz8`h;&M+24 z;_B#BjzW#PBYze`9<;nb#rIK4rYY7!i1sjsLR@9@^~=BXn}uV&1w&vHjCvXOKplyjvZ8Fj zH9hIeHi=(Vzoc=en829BzqPmX+5s-$fU?bxN5jL>`Q*j;tchd~k1>|ri9M$bmKpd@ z83q_pQQ@fMpaKyl2!(WSqtYD1;o)=&nlqFd1EB;0Ngxh{S7EvZp>C*Rd-Vo|R~rA+ zD^>Of`*w%GsQ)>D9mQd;3jvjKrH9<7`8J)^77z$VWe{f|^2t4AL6QZUPIIU*ZM7(e!Y20lepcZ)t*I+%C!ItUxGJv5WT_pP-OBLAZl@4dv-(oS=8;l^euk7S}^k(Kn<)8GrX zX>-epy^*toLNEwz_1_N8XzthvG{d`Q2;)l2YUzd2{^&>R6U2_t#3Mb#Gor9NqQBPx zC5Z%%t&W5Ag!@#iz-okz3tM#&qDL$%0pJiM?Bya9SB-Xs==U=rT)AcCOFj44Bh zqKYC$aQ0vpawE_B(;2qcEwc;bJzZ*vw%BE{c0b;%c4=L2nUkr57FVoBY3AXI zRanz~6-HRk=JVfgbratLam@R^k#PjcPy-4Dx-<*I<&+C8Hb?vh{#eJvsCWnLdX!I5 zmBV2WNo1yAmJaR^!iai^{h0Dh)@Q{6_XMYcQWO!h&}JiH9$6YSR0C24%955CMBqH@J&wH?@bng~F{dgvf9Bq#%Qe4QDUOf{V# zOwQ)uL10Z4$Y>V(-smo$;ag$WA^`3C9G*_TI6WD@|F6T~+3?F@a58-L#p&xI7}bYc zv{S*&34?$C&HsKG!deL?m+!yH;%r@9hC2ta1=zp;=6@M>7o5pTDx5*6)*0FkeB!kVhh^kEU@S-owDh(&%(RiOKY+`v&pzNcq*T8j)Lz~Zx`mOx!x z6`7%}Hwel^O#!x17#J#Ff0=U~8BlgiKq>>SXKX`f#27dH=C%DLj7AH|$p-cj^*! zyQylN<68xe1j3#tO@ptOYHl!ED$Q&xS(s8v7|&dXR18}P)VAEg#Ujj`IF3jhdrZL} z?17$%dHBSPgdC{$B9;UO@L#sxpm2Txv%bw>S>VYQkvuvGi2GsVmA?C~_1@RZlui)Y zA9raYTv2ZCnO^J26_e0}#Dz>7T>lBa0oVm0739pW|3GPif(&Y!bTjPJ73CHv3Hyf>5kSJ4LFwbB11Gg7ChDIk*36= zB&Y*i2Jgr6$Uv<*xb=$KhvB)bq-sDi{$T#VLC`bM3;189Nfuq2m>j)Q6+BpS4;O-r zWG4a}h|D5;lJf3u>$4P}xIARdTpD$PofyES*f0?`HJoA;3hGvlij&;vmOHwq@AVv{ zgMITGb^|AJtD0ZLev%K8W`USD2!AEUc#I+x1!V>*UjTk-b4rNMtfJ&6O&H}j%A&*I zChEVMh^S$#$#4nAuQjRTt^sY@oM&;Wyq8?sQGO>B0^ zG&V!Di0;4|Sw8EY*?SNxMSLsd(u%h%DC;R@{{MXLNK0G0=PxJRA2_Nb`}zt43t+{i zDekh%9Orn*Fm@lhKuAwi34V}gMX?hJ$RL@^+gGy#sZH<)3IA z;5L?xy{EzWBA8@J)k^=c6y*%?(ejWdXyv;~dG6?*-eA59-KNoDd`KW(uuj#n3#?bf zb49uq7X~x@{X4v`*Ee(}gB2li%)t@~xAQDmg1)9$rtkpTUEd+HPC9P7{#wtIYzmhr zEkdcTeP~d-tG=Wls3`+j)6KUEY8_C!?+p6ALL8 zN2$AF2dBGewg|?$X5zsdhE)~LYl8G0OVsWYKG#b>=t4kgv5iSt#T_y|M1z(gCK{G( zN%UL@{wt|wPAAohs-%=&Kto$1oID(N(ok_VJhdf8pjPRVe|9s zsI%kc26V9vXqw?Uh{&`=de|5fN9c=$c@FQBfk9QJVGvTSrLd2PC4o$>j*V8+o=6_R z#*Q;SUoH5%Sz!z1pf9cHpq19VMnNVPSa7wSsvrDSAL4B;^aa&--F-(s*Ua8J4>!6B zlL{>`XJAS_$4jp1f;)E2LOXU!QeO~}1gX&s-gh}g7IOwa7cH+w4MhyeOF*p>XUMa$ z!Mfyxy~STld#K?Z>_8QT7@Z{o8~s6i8erJ%J11FTMVT!4tlT{Zqsqn?t^MdNi+7RF zX;|B?Ni^jdycm|w+SoKbC;Gmwm0Wl4L8w~YtaWM9iXka<#1PLfD52j*ECD}`8&2+p zWNYxE7{RZp8b+*mLJ z5mFpQq%{C5XcUP^0l>3eP>LV&dES7ZYwop45eOG|tj~Iex6IyEHmhNx z#&S*?T0Iq1T3m9Zwgg$M6sQ~G<@?^n3q*D0pcOBBI4!bV?6?&MN-&9+M<5b~c2sD> z+NeSZB9&nEW!M6Kq^B4LqnfyV+@e@p5iRT>BS*M_`T zVt5GJbfu+C5CqxgF-^ei-QVB0&8)051IdBkd2Lx9-*^ENQL856K{%WP!Te|g{onFh zZ&AG|QJV~g5>$$LPJ%zOPO{~9uX3|bl2&rmA#`Y1RT^qO!Gr1Ua&p^?y`%|twX?6) zfnMmJjQVIj4foIr;?UIf z>+xhVIuUf^lhNOQNuBw=6wHTt5QBd?;^~$pFuo-CO0v-Y9kI#^<9iE=Ix2;bd|$eDQL0LM)EY zM=wX?lgSYOJ~};zH+*&Sd@{Mf7Yyz{Kbrg^7`}pcKYuYieg;=ef|FNY9FK@mLhg%kb!_(5MSSOZP`%hP-X-y%44B@FH7cxx41`AQ8xIgDiXh!IH` z2VuP;YSXf&bK=w*0~EZa_1~?p_EK)ziEegvjoy%95>lc-7~VEqv31JBH{bm(+BCOp;IShKkfd^1+*O$*}J zAb!WX3`*ddXCl3o&IN0*A#QJZu}?ucvK9lgy^8E;KcZQ$lyLelkILlYGCKnfn!6=>EiZIom zyro^P0JqjqPS8HcW>vmSpzlO4QpFVpa2Kg!4R2k!OUOD%7*6(G_zK(rS(0HfVZqFSy58UYVr zjR5RXDnkzQoWsP5+7Vs|5|Ov!n@*{@puqDlyD0!qc<(>V1bDp?mshms^hso)-v<(< z-0o6e9gww5PL*!wrJoB|seulSUNed_ zkBWH8vmH@o?JTYKcdz+Acl3g?UBy(|7aSHyqAXe-ft`REOEPA!-(lnuJ43|R@6fvr z1qVGG62Ue6dDS6*?QppUF&4-rqQfZ?z1BQc1kAjCXVQHgw9q}w@$LleVrhGl-;XKW1UR7k-roW6``vDjDoC5~~PG8N!cc|fB%1+fqFvf(Q zIJg_!AHN*$8X<{WT<0xhc7-UkIO9-Il1o3X7hF|IT9;qcsYto!yEv&xc6yZY1*&6i zQ&v=})6%m_Gpf-tOF@Zl_NDskj>TJr=R#a{gO!f{Pfa_gFyZ7$gT>J!NV>#uB8HJC zSIwVF(TTdpr6JpE?O0K3g1jn)fM_Co3!!fynMF+c`cH4BsLI12GC}m*|QZ>_`?awpIgmX$!NuKyiEOQ!kF(w^rQltHqHc_^uVz6(AyI%TjWsM5vfTv62G2Mq>~{ zA?d;ulI9%rzrxmwT6G%0VAQCpd$-H;r8U_GE0I5Ni3DSv=aLQ7RE~==WcW!%>U)** zI)#^7+HU{Ny`48#;R;9S-P$@(&^Dn!jaM?k5TP_%)E#KMk@YdR^CGldNQj%&af4(O z%@}(DDCHGLHQJPSn(GDgLw?1#>u6OrFk;-RY!hPfS>2vBs0%Mc%98kP`bnq%O~rWjEB3A96q!$3R8UBE@NtrC(;#(})`6g>*nHxz)~(LKFF+f{3#)f~KJh&7@_ zR1brxup?pR$y#1?>~NPiwY{4LoJ4S#1kE`^R?9nrHiNAxZ&*m#K!r70v2)GyD{p;M zI?LX~VCKR*%`sd>#qv$&UMABTItNy)zzZZ0kSk||iJ~OI;a{>vmSBlcPsZ#y_A`ac zMZhkTNN36aC2e&uwBte%jM!wPTp%|oeAOjeXZfYNm<=D=iMaZkc|&ihhl|+WXvKIa z$eXR9Q>AW*79{(+;&LOlK~U)%b$IA;;2wTik&6M3X3k34nvEmhW3%oh$K$N-eom8C z=Zd&IX@Pw}GQ{U%lhDpE&9DEr_=?`lsVj%0x$)f|AA}_Cyu6KW$5kc%5%=*n3UcxN zspwnud*^8;=yhj;99sfv;kob)=iHgcsOS6sMTE$mCp zK?oGZ92{y}rOoZG_7dRR;S8bbiCnU7Ehrg~&1Q*+aV2M%x9Al4gJ9ca6>-W}7|)h% zEMZ#^LcLo;(5+aVvT}_*>xyWBfRmm*#Mo;}(nU@Mtlp%Nqg}3gJLdnV`z~H2jonn# z48=q6AHT?1%*z?<0nq^~m?8A^s2V>FzWR(B9u{C#By1tOZHo$8?4vABtU;*sV~fi+ zk*J4kS~?(D2EnQ$v}nA*n0zJH>6i-BYp14gpXik;wk^>&>bVftAmVPA7QpCYiOO6u zmLdI+C8O#_ zW%m9r317n}1S!cbNhfTnm&H5-n;(SYB@eKXQ1<>C7GV#IqFiG*8TZ5%uc#rZW*}H? z#wyGlDdp>EOc$=;jj}#|xHs5y)7thQYW0Cw=Mx1NV1b$F8uMqL<>(;R8|Vv?^ekp^ zB?@+l-4Aq*l8)lE>gn5E?}diDsigXfY4&%5=w2UAd)P*_klKG57z~|y zp=HM=X?7>FDb+Zr50X}Q^dK+L(N*gBM2`k7vsBpN7{9~l6*xp-FroS+UP2_VZ-aRi z0ZA8|n4F$2cX*c$gI^EL-zja7X%`IgB#eYxukPK2HQcMR+cQUN zqoZ_2RL7rHC8z^2(5o=#80VdFmc#@$mM^jq8r5tS27@GCZS3>4Kq1uaCBJC`(@OD^ zW<9Du7v`JGPLu z-0}`yP=;&G>^jpro!XzaD>g0WILjec|k_4>aq(pZBkrCS^)+@MZj9CCWJ)gxF%QTusI7BIN#); z-qAh1xkH=R+t+QjNx4~8i6qNtNGzH(f`Fa`>uvakEmQUe)5X}l0MA(mu^=wOOV{`> zB%BcHI5ta{6x*oz}{p#l+r}~>Z-``oac=exR4WC!IqDW-)1mL7oaIO zvry55ZG^jHg6Z$|DIe@*`th*xsqk@v$->IY2~Q@2%2LT`L1c+1rXZCls+axurW;ZE zD%kbeTdb;<%1rGLBy;?>uH=8caroqrqyLo(96ADbB{&)&9jA%PvjUg{B?V?IuwNBmoH6aB}2NTgIp~Qjc#R*f`vU@pq|$@31+g0EeMTi zP=d1%`2?Fvs0dUd*azHD0Iq!OUEbJRdggL&^nS}7h`1JBpV`9Zf|kB!LZ|qE%-h!| z!>@0cgWd86ewf$WbCoG;_Qz3`lS;!#o-xL@VRnaJa8{)>0(njgGwaFv^A5_Xkge-Q zK%~oICQhaBHwKM_0Ig+VHw`Z0-fP$^J5y$%PvV^@ z!S1{tis$%#x9|;q&(;1>D_fN9%P4eMgDl0gr=cWZiLwTrZ+!CGUhUJF7TTRgXMS#G zh2T>j32o9TEC_}O->}hTedtBr7Dq1gM+@A%kPU^%xWHx;*nCK0#yKk`?kX|51K;90 zAK}fzJ%_H*xD%;I@-Zc@&(LQ8>6iN(Dtk`$*7>_~GfI-_QoQ6S;!zQWCxPFaa{4HR!bsShHTBywe*VJ^?4lK9k5 z_sK@9M%E$q&i?bTbT-BI{WqX(;BjlvHb||=x?r8){?X{dq#G0_Fuv&s+ZW#8I#=X3B^MkV+B%w($5oz7+_31=7<2Kq*x(I|fouFM zQHBeWWS(Jb$1TVBUUCnU&!{kyx~&3h2mrVVw`+`eIi($zB5Ax64Kt{X5`32%oDy*7 zUEPIsQpek7Kt4K!a?=jBr_*(ddUO$-u1q4A&T-!&aMaFf4%;)HwH4p(rxTR%9q5(# z4eMJ%WP>jR*F3ElY;M(1EVQiw^6x`5EOsp_cjsA zj(JxHu6d4HVdOyu%~B~qWcPlWFKP;pn-C3>J{V#v#p zR?8sE+kel%JJb@)^))$*HW9rnRbABiMDWtCG9K3oPYy808YGYmAl6ty{-&j{<8S1x zycAllj?IcYDA>7M#?Tvyyj`by3#?r`MC0=s9ZW{YhB+w@dV?As%KyhUJ=>*hP8(If z>54yOpL6Ne55L?eIIKl}yE1;%8MM+Zoq8PO@(PJ1rO=u&wj;2$V^n%Dc11njklK`f zI5wznjW_gl=36uakCJd+D__G#?EdKEf^tB5wRVi>*jVzCs`HG>Bn(N!woWJDW0u+b zzZBszd;ibh{WobEz9}vZ>+t->zLmzf9Tb0$3@(-0o`FTB;-$I38&JHm*Psu;&lRl$ zld1tA3gU5DY1>>=cWe}ex?ym|*;+t3vZ^eQ-M)EbSU0`3HjV$Wjn7?vwEKA1MQ6O{ zw5d9%DsLyx54*%GuY8o8YK}LYF}Mv_=OVu+g8wx`pHg^YwxT1BqZMZFtDHC1uu@^& z1itG!3mF$}d4rpY*N8ZqZUK%k#!qVG-dX||v_-SdFHkeiSWX^z%f}?@Q&KV}R+MIu zn*qz1WSP9uKfsb%yW3lPGa7b6HbF~Hgt_H^}jD;)a54XC& z7zT+_OEs$Uo=Yk{0%=$I6g?Hhz8OQi_2pg$2|Hn-(lM<=_$wNk)fn*HjJ^^ctzm4g zmpm~tw0LcBri1k&3z9Ipgq_sXUGwxRORfZzlXtd*xYYJMV-NqRhw3F0b@4eX()A>a z77NV7!)7{$XYG(zhS7;~&GS-(9?VJU$f}m!vu43yW&yt*W$0VOLc$vtm0o|y?e0Y)cO#`cYk|t(*{@BvF852Duia;L9*{CqXJ5-Hx`~( zhgeu3?lipe3h6u|TN?Ea@9Raq+4CM+FyQF=U$Ok7qFi)^&FrrKxZv|k#!Jv#afOPF zQ7^Q&Td}3ULB(8uRe)TI@w%(54y3`?(_i-G!DJ{4yb_dbl&klz%7G*Cy05Rwnjmk< zkwcG^OI_yDEtExm7+fNR>z!p$MZK71vo~Shkx78m;2HK6tZ(9wLOYYwYQz2@iN^&32&niSk;Vf$xMwDb#++L%<;3s~z@B2BLI1w|0LSu1&!!j6m zo1q`^_1vPHP&xPMa7hA!oy>_Z2BS{`nL^=8W1T;^-n zhzp=hu5hiv_YU8GLlaG+p;3aL%a&yTIQ26B~d;1t8z3X6L+0)azlTNcMp&&%$t@d~u8xad9ShkYRi-UjSqwnU$zqw?$BDud5 zZ3>CeMIk&w#73OK@YZ0J<=Hxd-vT&`GN7#7z(Nox4>$%sKe!y*joGVrLLF&w{OEDX z>?sxd!Ji#6c5gME0sgQ833kbD*ww;h4K!d4OLiI7i;zarVRDi?cZ18hiZ|&c4>EkS zHMlRA9P~simgxppV+E@_c>m9N7H4ZDe=>YCYYIWUe243)D=elJsBk#zhBxn9lTm$> znXoQQt(67X&Z&Y8ZmehFIM2l&PShKI!NvryD^X3K?$bQfOQqy$kp9GGuiz#lLbrSH zq27{Hk+faUn@U}i>I1~h41O&2N)Z?;(oM^+=`ve>_{Vx3dRM4PvtNdUiR`FB#O#9F z%2JY1qyWVdekz-^38(>L_iHW5mQxnTQeAeof1UqmAuWzu>r?n+8I-s2of`+JWLmJh z-Nez&UHQpd@!&ykD*9VsK|s#NTw2UI*8N)HrQTD$d23g7u*PdtOq1;E3Og6GM0CC+ zcrmsR$`E@CTcN-{rcrhMhZ0;f?CW%unc0`7XC{8%uPY+z5}ke96q`S3F>nAfH0J6F zY)aGu^#B0+{d-UV literal 0 HcmV?d00001 diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..85440ceed5531f242a912f2661aa698ec3835a4e GIT binary patch literal 34620 zcmV(?K-a$?iwFqs+rwS}19xa|Y-L|?a&KpHVQpyU`Sxm$<-`8T*A$-j!>+6j;G_So^@3ODM70fTqMqcBRY<%>?k z*=|3KlO4y$PjnHyh(*J>6Q|RuKZiSfHyEbw5U(FZpZ#C|$A8oMb9h+OpXYjiTD>@o z`h59A;7r5FVZJkAQQ!4KY?|*nJP5*DS6o1EUw%k<-*t8!_SMu66Zw+|H(o~JFmflZ z)28eG>N=euVLl(i5HYx+okro*75_83d*()C7^QTYxMSvo1JzaW{sacA*j*Nvy1W0h zrn`H3cdxjYMiFzk4>J?Q=>+=8#k;ucK#!TjcuD{^0&@Ue>q-$7;xI8aR}@D18nV|{$`9G z{KOL1caBD|I#@7-mu_We#;`Mc90d$5QregJ5f}!e8FMLZW)YH z;JL6oubffc|v!e#krMD$%3+_6OR%*%~@-9YgD`SQ5rknBpp8p6i)| zymk?pE4k{p5`v&`V4bn|VdO^m`o22_aSiXs=IDZ*S{FN)U!Jwg4QmG@Hz4_idn_L# z`LJY;BiKD_uWN3lcrMAT)6Q}Gs`c*Gh0{4dzUf|Hby}yeFgt}K2s<)mzfRfm|8<=? z>Jg{h|?J=_a;H5mGNkxGk!p@?Xc#oDJxX)uZzzed=VGbs4z?Xl<)2VQ}Bo3TeZg1O1iy@XQyG}19O(8o$r3}I&e~v>k zN}>5~uH1xjnxHo+bYGpHvV-&N_}q3Eja9f;wRQ2FdiMB0@8nCylgJ%H51^wcMV8<5 zxIcoqI|(EBfbwaPQZJ%_X5NKO>%3(?+y?D)qXgOu7rF!JnAxeA!Qni@B^ise7k5$N zX%3vwpO;&{ zMmcHN1Z=@0QX_dHw+|fW9;nsJ5AXpRkw7Cz5|(5V>;_sC`n@oMV$7fi?&_s(Ej-`N z_{ZMad2EvFSzD5zWgW}9%~$RK^rM9< zrIUCGSRbH5=vo7+5zM(f>ujmL8!v`eaN2u5JOsA!^2Z3;$(-{vStuPqZ7S4djIGpo zDVW=)dRVcUg<(B$Z^MZ7K_K{Wp1Zd&MzBWdJc^w}*J~NRd-cL~J$XM$0BUKX^xOFw z5%#1`)qYl#)1W?rIxNopa56=+4YWeCPU&Rgfw%lHWuPJkeL8*oN5V40pFv%mgz0Sv za$f10Vt=||v%<28!PW)2sJ5t-ksr+YIf7R+dNT;{INouBFxd4NzL(*9QHIwkUK{V) zD;B^G=nLE2D{O-ga6;?-fhoA$n$lx5_(|OsI4o{~f?1RwhA>=gz(FyDgMpS>oC@7x zk?U=+B-03Pfm0Oz>bdV-GGNuu>im`K6ec~5lZcb4<-oQIVi>7O{x~oag6v$+?R~=> z>%wDPj?0 zu$@^v4JYozSQl)FkbW*aMZ!k5zk?TPQYSvdZ{-L#@q=VI3Q=x!!e&Yq340} zaQDr82KsF3fO3BK`zZ9n8IQ-S?Ru6;bFn`1^(0;y)4J-V-1p$BH3#Ow{_{$G_7MKj z`rW3{u%=FcZxaKBR02@DFWTXYBo!ySU+W9xp$@$cNJ@>@Jk<`0VAM zC-A9Holasr-#}i0UO(Rab91waLMp!&C))2QMI&-%jNRoZTDUK z==FK0b>3+^#~0l<7w4_Jf4R8(-BHKsIPY4Q?YHtrxbO=8dv)R59KCCIug^P2*Kb>f z<_070dctJtqgYdF_>9A3;P>crfMa)R$|S_Ys*D7OSkU$}+rmp>W~-R@BfRHEGh3Z!7T=A^XSr>!Bo4Si`t zlb+1QDSyjA+lH0=h=FbpPAF)U2GX^QW;R>^iUJ-#6Si1LcRgIJ+1}+2v_S1%cDd% zw`fnl0=?AtK0j_dR~Mk9JIC#- z%+cHy3i~J`xpo!eg6lC5$}~o7C?r)%vPN1cxOE&&eKk6H%J6_i*PSqVl1tMFeUbo*H*QhNRnhU}L&7uqUH%)<+c&u99J- zd4CZItq{Hg6(ZpMMNQ1mCp#D|dg-0TuHm2VFyj5xr>*To@FF}$vA64VJiD&Fa$!Rz zJVIx(8H|_Ca$QM_!M+5=UVIEtult}(+h3=QG(!BAnjKd7ZMM?c4gKxEWKjgmhEYrm zHCh?r0y)!*+_XQ6X<5Q$J%;9pxGZvxF7|g{a(MqjZplJW0dEQ5SRT240$(YI+jiaW z<&|rx%e{5|vFj{g*@ActK7Q+rNe@e5NrDmt$9o8di;L?z<{!UdFh! zvc;!ilHfrAugQGrE`kR@SEfI?d@CHQTWO~u@on^(P7I$?>HWplC8)`uQqyf;*{d<8 z?OEj|!7+MPFi94ic?;f8W}!2t?E*%l;VxABXaqYbao0`H!HcRbbnw{KUwj1mPUhbV zQaIPa5P~_8rVnXBpagLvc6iO)3C;y<`zVjt&>)hos+?SS*rWc5*G6oJKdo_IRr;Lw z$~*sL`M)AhZ0LZc=q*})KjOnN<6yhWO^ru1bYVfK5r)4b5bSB(Pf{w!jLwhpy=Cvy8ap&ab{J3-FoVJ|yKOePE+vi7}mh+|4I=ejWwCMZG z*41_AsB_sm>MZ~5e5)F&p=8^w&Px|g;|U>slq_$$ILrIOa+cr5Q#fZyoCU%klY+hR z{^NIvj5R6KFL2+_=665F3-;c5?~cWfvpD(q-4uOt3$P{Wun*jHEKc3BRyx=M4sRW3YooKnveS zuq8MXSOh;g^-6N%Mw-4~%lN}Fi;H@w)S)NwhFBN0#tFup|OMu8>%(a{LQD`#{3-as2+hoeTGI)DHAKgKQ| zzfSw|<=?;l@6PA&^vC#he(b{6KeIiQ&6Cl_!NCJiT}MfVwmP}mJ!KnpV29qEzjA(gaqi?nEjZYn)=l@V{1FZ4m+k9z=d|+*{@&^g8McGn44qQ6 z8>K$NS2Qt|k58OMvAuBO@<0Z^bKF+o#E-zNOy`Ku0uvc8RrpEf{PdUQZ{?L$07_;8 zM4L{Tu{iCJ>0Qtp0uJIz0J0Pu9pQd128Kbi!N~IJK(XP&Xox~L-1l~Ef9Hm<2qJ;A znK`&sK7P9cc3nIUv6b`pXa|Z70wy1HZk$BusrdNqeOqjpe0D84UO@(JPs=yMGc$0F@`0_UE!7>3=1Kx%neb}2i8Ne>=7*n zO2P&jy3Ti3#Cg6syD2wx9T3#ioh)%SgbT``pbG6mh$|)n_dEC#`jvNQO^9J=0ls7Meq&GG2UBC9v~1x4hKh(@WA_$8KpxYgtZVeef}kaEPH*`?69Tz6 zz1JoAoWkJAR2vRPychuz+Nwfs^cJvEqPL z=!PQkTzaNUg{43U5exG`VP}rU%+$bssYx=BpgZWn3d5#DCu5%gjiJOw) zzfL_F!nv^g5M{v}Jw|7!EL#f8ncLS|8p5&0B(_MymbzboE~e~_33{8CAMPy-H^$I* zljZl(SOt@E?I|9+sGh*D3Hb~vRILrsvxWv{qZhvITpWM=4G4?Ni+5ih{i5Z(buO>o zwa>o1``xM13kfU(QU^i%P1V8;{h`vGjEhyclTFmo8pmjP;;i=4y3u&cL+|6aee{kl z=6$dLHHSsM;4#b=M&V)>zE=Q26zY?CeD_Ud{^*u=)cwXj;fNQcIT~1Vn_iRs14LB6WWH3(95||MWL!7%l%NsH_TUSi(GP6*XbFE$#Yw8OE5| z30hZxm(D(4G&kzyA;$MB{(7m_I4=NopL&ea;w%yx%>->a4pr=)wqAH|Zv>W>56A z=R&_SBw(JRA<{PK)iL$bMSuzRY57;^l#58)K+mLb9|l}vW;$ACUljqVvw7~Tc2Vy_ z@sEcE9Q*WnMn^#`Ty@Skj3*ooj|%X2!@MlRxU@DfHI;AYnqK~$PW z@=t#Rs$U?afl>us`A0Ha{t(q*1V3$9){r5#e(^FA_9Udu2D&0epa~hPrZ!oD_}su7 zzVW=C=i+2155fn~Yz#3-aK7TQiQNd!-&)^7KE#)x_rtTY5e^=g`?#+8_I?iC8KY9K zvc4`c6e1txE9Irv=B_7RiG9#aWgp_AVSP$stFnv3gQs}KrtHYCa90p@B z3xhuFIRWJnAzuu7u?4jM?nn837`cl=j9G*Q-p810Cj`xcWk>ORvQ4RTA75K0L@=H` zaC^~wEbyBPkR{l&?ReU@(L}ZFp^C-kE8cw+c4H^P{jndc3+Hf1yaI;9N`N{$<-txNf5SyGy&8&3q` zNuKkJdz?boxW8eXMGjiF$scZyo4GnR0NVh;myygo{OK=*l|jUI`^rQG)PZSCY>^G* z^5o(iG~qVFX^sS(=3UEaE6inEp=sKPrg_)7I5$+Udx13!$|t+!4U(zn3K|f#1PQ?} zvC_D7Fjw=DT;$A9s5V;H_s-JiI^P@LDV$j$9zvY>wg%i+sY?+DWj{uP*QN8Sj${tB z1*zk3&2xtN#LQ>67$?i%aDUWjzy8sBS84hWpB_|4Ko80XBE=&aNlkbMd=_qE7+e&l zgA_#`3}CilVQhP#*u)G^Ea8YE@*Eco9vBDmPwzOh5f{;5SkbX`L!+0D4;V(8@hLEx z%#ZO(P-6Q;d4Pu`hNF@ZN}XGrPU7LwM`yJz1o~ic7-~E{PanT14ns#YRSbjz?jWm% zoUy(z-|$xgW}+`{u8tcJsRI__v}WTKiy^|(X&QoEqYv)pomT@JsUR}WQ?nsC*sC<4 z_YAa0``S4HeO%EV8Qcx}kM+6$EW~OL@?H@%rYt&G>Su7i^eNaYaMJYUzVmtKtg~-` zG1v9=0Vpa;!t3y0Z@*@%mGXK!*T+s9?AV~`Dn5Afto99sW53fnzI3iA{{dD`i%0#) zHT9=;Tc&!#aLbes%M;k^AOT?6q7TvWT@Z#&+(+CDZi@l#{d}jSBnU=xFACAhLm!?b z-a;`qj57Ze>?Agm`C|frCBe-c9e=bz<@Z7+?VzCXQ%}el;g#kI?LY^V8yypr-wFyN z^q-WObVBNg zTL;tU->Bqfma$x3FTYPzUQfA+E0ddzfV&-bLyP9RLB#liY3&QI9br$ z7%4~PTaIH|*~l8WhS|nyu~@=aYROLoO_wQ7Dw%Nz+hVaxJ`Q056VU*C3x_;9Kh8qh z`!KMTZIwT-_Qk-Wdr`aSo*Ikp#09nGDulLdAovdU>eu(P@)qi56bFq&YqbPG(Yjx+ zjR={eX(AvaHFX01%7iL{GyJ)bS~K)2OQ{z6b1?Qug~^4P600ShA_QB}%~blnJLMR} zB28ll?twPB2{jDFY7Y)GPs(AX4P}kd+Ng9V(rhBsy5(^|=R`@#G!-Je%Jn5R#dEOZyrFUd7IdY{O+bo@NqdE6 z(GJ(NBV1FvgD}icBL*yRT{Eift^KmfEeL0D6bb(dqtG9P&AV-z#*4lJ+NdD=oOsO9 zKv^tbl=4sJ}+AKx4mDpD%|7 z3<4Qxzz7*Di%6{IF!ZJJn2n3a`b1p=J9w;=-f0R#t`BoJl!tb+!LiI=v&uQUUn*dE zo7o^pG~#~^+*1IMbTVW}BkYLLoM+W%&R$UyJ!8oz^n^T;sHFyzakyu$R9JCIIN2@H z$VzGM#O{DO`DQ`wJMH^$#@6YsYgR*z2ejnF0L2WJ9}1Zm&XUB*C}(v$2h=mqJTUEV zT-D4yLxgtDJCsNlmq{jBzFZ)nlhR_mu_9Dz1*9U&2 z2!p0ADXjsTR)Bys(rc@Qa2c_0rCK9%H;fdHmMaz;9iR*>+|e8zK!&*4;&WGS$LbZZ zS6Zlj`Cg{PWz77+u4dt&M}pqkaFDIh!CyL$GIVWUN7pt3;2NjP`Z#a*@(N^P=lqqU zaVE6RkDa%jD?Hp6$Bt8U3rWkThq*0 z*O2Ok^StDGo*-5kG?94^owUfBF8qH@UR85sJTGNgN4V963dH4WQCev|heg^)S%Fc$ zlI66c{5J>=kR(%R>(J>ifp<4G&TTV(nxyS)Bj3kps4ZI07G-x|Xr5s^SE`+RbVQYc zQ{>h`akK*}9vIT|%=uC{+B-+r^1NcZ_cEaO_NyKL2W7T;&EtgLh*=o<-frJ_<9JJ^ z{9dJlPqd`uW|na`DaT=80wJ5YI9RhvIKpu(B)V+|=>~Fc$N|A7(~*nu z3iS30pM;*Ii3$*OhA1Ay_!T=m3TM>giWlK1fHV5|Kl1whH2-qaAN zQ@WqlO4PD*yAYS~=eq+0RC^dA<1xA%n)})7hOD&WUrV){^kdY;QAJl;7M4IDHQjHz z4G&%}FizsL=#lx5`bNgcvG?LdR4w$>HM1M0d$OcV2Q3FzYR#vvJ5dgqY3PLL> zm^+nja&ngK7zYORgB>UuhXPBo)Mm@)Ey5W59;iXVR(yXJ|3ia91=84_>+|H z8LW|M7{tU4khv;Kni@3@+wPcmsmK;IYoF%mz*OA3~-}4 zCE1`5&-~TNPOg~jXfc8?n(vU?PWOr^=a57oi0!#kS_g@Ub`zxD?Gp;XnQ)21-zh-y z<&sDC-6^c_8s%h-*3qQ0xeYsF1Xg51tC@nH8Yq!exzyAK?pepYtUc5?`}T^?KH&zy zXA|;6RLRLye(4^;UmLo3H^{JEmn*x49NP`y`<*;bOcO}bXVJr%QrPp1)MdHGH9QJ7>N} z^DR3ff#s%47G(&zm*W&49T!9G0%1kV0)|W>DP5{B! z$%p|_l@clsGbkLxChp+_7gba-=0Wbl`6u9o3bpGiE?SZ2Z zvyrqC^O8J{>W3mO+WGvbeRV&(z#JE=X}E5mI0M(Dj9oDUW|1H(i?JPml7u;fQIRur zDk%Aib|To0JVq{2PcSG1E*uaP^KN1_8fo&jjABHH4%#HOjOf3Ty#AJtMYmk6K+MoT zy_!?rmOIR>uVr=eUojMQeKDruywAePz8G4jYLQ9AeM4eL6}l96xp%*5Byu*c)=cRFREt5%Z9gO6VVaSo3n3*Zc`oaP zpoDpBYA+cBZFNTCen`@<+YCY|0^JofU_n<33y-5_sH74GpyNIZ7L6RN zn;z=1@Bn9>1w047Mnhprb^K+J3s+u}{M%$yr3(=uV4gp$dJJ`a`Pp!MLQ zor6^=wxD7fYs6mDM>^9}Ovuj|Nh&q*8jW+q{QE9awY`^x&MFE|szMK>a>!C9SfhzM z6Oeb_VlLA1!x?->!E|J^`}nU7G!R32*@TR07-0)Bqd=>*V<;oSU4b0Oq&;OEOy!1e zDS~7y3#(3cppUJ0uP(GcVi!5I$XyJPFfr>Ak+9`0VzH?IC84?1%-h?Fju~>sZsmbC z3s$~Ma-8*riUP{4F2-F2&{+OaQ+i>VSfpvaiO_nhWqI|YGevm=EuAb0O{tRBnQH2X zVpFjixhPhVtLS)VgVx@|o=2fV?7`4z$?{=m-W&47?wx2WyHg~+w|h-_BCO>5@kRH| z#d+)QUs}$~76|$C*3FxXx2-Iy_&y%H&J~@X@nh#De7O6Kb58Ygk=ho-_RI78iTRD3 z9~ArSWaW~tGq=b4P&S-S-Htjl#n}pHQs_4_Rd|C2 z!I=8XABMz!tRe?yMeFSDEe$1(q6fWSe&`oRarwV#j0=t2(r6lu-K*Y8`{f+7f?T9S z7S|8Ch3^O6 z5rSGpfVK!(k~3cq(j;s04b(DGvj(y7n5;@GBxXa9aw|Krz2w)jbG0(}v$S+-#6}m} zEvo!xod*G}5i}9mvVWWXnGXfwdkdRKY?z&VzX)>yg?=nFRgO~!--7thYW^519Sn-z z1P}yj#&@)|jXW5fJ#3v9+HSqJtU7=H1#2|VZow8xGRz%bAcLPvLObL=3Bp}0?)c+; z)?dK)!X$tH@!LKOKU+*W6@8qfSOnAqQyV{q9$wB}EHWR1CH`p^v{8A%W#-o7U~L9Jra1J7 z>U9J*LP{;aom{@-Rj_k}DT`%1=$`&46>xIrB@gR`+}2jQA^V!HbitK9&z8F63f{^( zH%i=AI`^NJNnv2*G9CtQ7^i~W zfuUR7tE)zo#%7BOA3LDycoYelEPsd=V1F%t@SMBf!@ZvW@mo)r)_(yf^r3T#MReRhz zf?7#Vv0u*T_>_G+6a1KW-^}3|?o5XEOv_0`jlDWP%?*3{;6--fQP4yD*-gJf8q}a) z0gzbNJHwx8*SnMDA6{Rz&p|c5y1;IM_K^1zT6VJhaTF0P@D(EAKzHI`ePYQcMtF5m zYfbM@P^^`-OBJDpOx1*f;d#rVv8cb+w3FqN{%Pe#d|I?|yC)s!-X(NQY5!QIa@T1I z?;4g+;=V}5+R@xQDo2MrZN*9yh0=_MRV;J5O3ATn&P_YtLYoH#S4>=}p%y#cBnl^B zzxQBL0+uYce70R=+ zf`cxIgY3&_6c8f?U z1l3^>)Ww`=tlk&1iSXFR#oQ>5Y53Rz(2EHWBN!h;x^l`=oO@cILKveHC=t;~{V~X7 zeu}WHIK=`cSeZbX z!8%PgUNybEnL%zIt_c_IWn44)h{VFcbI{$X_2MnkCC z11%Z+nG$Ke(h$mFAX#z^0e0o+&%9jUcq3h5qXmEW4QSs8oJ&#j#>7fgp9A;3&!+KW zE?3h$NwF>>T9Pv=L+X4E?|lsC7p=aZ3HuRMat{R73_I#`xy}&d8OHqMw?pApgPXCk zX%8zQgM8<@#^0lG@s1-@eRR5IMHzPR753m`qt*s8GX>>CynrfXW_eLyQOJ^++Tz$!n>Q3_|AYyn1HPaM#v=qxeD2{{ zF+iJ3o+WJ_^cY+;Qbn|u|Blc!I)kt**7kPeeJrYe0%J`U)vhv=c&_gief|SbxKfdN zp{^D6W0rrDks5TeeHxSM{?QvuuGQQbbFE=yBn>c!LqtAYUw4prtQf&TFUmuNa10?^ zQ`(}0u0CdQ&kteKxTG8tyCC7!%Sy1@PesAy*trF3*#tM~yoAUVhp;Z6-wO35uh@5A z3_#tv&_zR?Ef6J}8T&FI5{kGtlVR|%7zU>TwhHC01^;)34se@`m0EC7o5^BJVB}tFXSk#LL%kM|@eG3$ss3PPq&=FLyU4;Kd z2!F_8%vaa&0-r{@-Y0Osc5klif>r3=0JPhMR32ch$_N-861Q%!a6MCwu8W-*>#dUa z+TdvI2>T5ci$8)1kjCV~31H$eS_NAKV_G5>pJHXg1Uez^w^|SvvL~D*wgs@SEav zwSV7Oh;|+Nlrh>;2b%TI(CYW=0HGF+Ycx7gR}!Ga9mFS1Dfv}8>P*P3;A&p8TPpJs z1)8B~YD~hrAALUKl(afUgWW?voMtAWcSv}|oFGe}n8j1fz)`L=nF$uSQRv}ujN2$c z)cp*n$peKS{u6JIn1&$Ft!D2Htv`1dW!0OBU|ZJaBz}02YOUV1wnVtCxPE5?XSu($ z-Uvrm?bYgn1VS#d{$BRQg9SpMa?_LutE%(*QT${TP5y^#tmRazEKv zGJC^~_XvP&2*Y}isCJts*xcJD!pijYS@}pV1v62Yxiee}&Vs}zNaQr_kw+5IkP-6n z-EPr2Y1mi|vWi5A*i+0NT0RkljHx_D2Wr8{%aur|3Q_ zYQv7+kk=AJAjzVLFn-B$P_Xh^LC?S{A@3$7;0e1w*m2%sMn^>3CIcD&k~4n*4BbY)+3vXI6(Ur+CCP&^5 zqS6x}8}7hEkmuL)cszhJ1uZWRG=XUg9NAVt)sT=j^I5HWw)(PA zbSfycXd-r;5(FdFD9Z?^=t>?U4UhpEqynQL`~c1X`nb_lmsG^V`f?`EN^WrJm+s_# z>0g9M0DGOYOm;GE4(IciFFU)xxHfaMOL8tKH>a++ONlUPl^d(70A|^Vjz@@2nus3j zJPcHHu{UXdjVlJ7pv6iQM<4?lJ-b=5bInm%thP`r7%5ef+*J>XuA9gcyI9>iuU&JG zgW4?SWTh#dqHoaW9vO9BsyHpv7;#xb_e0+fnq5d}=PUm$UP^l4_7ukry(?la%d!@KbL&TnLd`cuP@4`QT3Q&xgV-qf&L1tD_N+4mlJ{Kb#I%^f@N5B zi_5!!^haKu*@eo*w81rbG+$P48~STbWwv5pe|c^K^K<=`4~%~Y$V zhvl8BDx*#~I#Hz&Aa95cX3lwkauiiRC>^MxshHG!|_k`H*JKw)O#= zrT3zZeYFqZ@+}k6SF$ucr3zYQb(m_hP{meF!I1T(iQzq_(wz~=pDg04&lH%8tfUyz z{PC#!-zvV?={(~i|^2#Y!n z4Y!FE=D1Ry>558)uOd8AT7>)0O2=F`L~xr$@T|f*Th?Dyu_HA!oL#j#T1d znjvP&r$?oDIVcvNX~SUw&4b253;+K0@7h-_=V;{eDE|A`2BQrA*ry-Yz1hOGs<|^& z(pqBg*t|OKXwJ=t@spbJ+6q8mi3PYXA(2t}$^wK^v+)u83deNcr~1ced}IY5nF5=v zhoBgvWrK39Y|x2B=gIoFYrM*0i6__Eco23Ah|kMdmuK+K{tx|SOIk#p>Mat*|({Ipi|4|q~Ho3f~nIY+>IBY zA`j9%!f1ff^cSS*a|3x7irN=uKz|!{gINosL2qSYi9($njg&~lb`UAa0xr^MwSLEi zmZo1Bcp+$dQc;d%E8R8=QCKM%pCEpG<))K?J2KU!3gyE<72EaV;dx#%NQ{jNWq?>M z0SONZV%3ldjiUI*ezF85J!ogTmGYxvDh&9o-Bf&<1yw{=7-!OoN!gGw=;0 zNGvld%+35V!bwE0Y4hFkTcY48w`qfDx1j_-t#$wwmX;kj?-k|Q3Cdkrn0r8o*2sAa zQw=@Dee*UP`ItCAGPi}OO1(ziwae>zm-9(vHPQq$GF$UA&{ZSiwE-b@cK5r>9r!g~ z^cQTn{J|H{b+{TPXcG26ehcOWgIy!)vE>O}X}J@YSzH?~otqivmx$yW%1M3?JxdXM zk0l2Mf)lB0B%Zjs65$5RX2a(GW70V{=$|2TD+8ONlKCf6tWMlsFUoW#qW6k5)k><6 zt4Jn;bl_q^z6=bN#oE@XP?kQYR*YdRat086fnvdOL1MY8w2_xqk%OVSpgW*E3r%SQ z57b8Du8vRN^;2(05XFU3!Y%(azzhr|y~nzdiED6_R0%bM^Xgf(r}()|t8%V7krOC1 z&naRKhhlDg-eXZBO0meyK`IM?n@AAhVNHnG>GRaXym&MFR=9laIqRy{&%xtf;Slt( z5IP1$MWI4U7Jv`Pk?FBg!j<8(pYb4A<+R^I123f`i4r+Lt6R+IFstRrx@Q|rytzdi zuV-N-XwdQuN|juhQ=lQFUy-1}M@fPvn`dZf$#$N3U){}4=je)qVaJKxw5Lu8l?Fh> zGN{ZkDh?M7E6$24wa&=Ouyb24{{Sr*MvM>0csOIw9+cQx#emEWovop5d9aP;W5bSV zS!Y4W2#if4=z|VpAbQLEFPmD(1^E$PQsyfvyNgq1s!fk+3;| zW?>*+=tUDxE}!pqj@zy7Zu{z}=+%Z+Pcq#P4INbSeg%pI**)`LeFnOr7rA#oQU){r z7=u{yCUa5A8g7RFz&vu^Vu@Z*9+P=qSdO#ztJnhb!wYkpQMshE1(ir-5OhA@fAZux zi1hs@drytcYSZ@Bo6$ z)OEDZk;Csjtn;up$KYuNKgmiB!dR3xJ*vrWhc_%gfa0mfnVeo3-K)}uGEFtkv*Af8 z728(os4*5l=SetcCT?JLk2ad4MrU40h6<-YrN&Xp=?NS}Xp58d?Pk?*q%@D|>sSc% zmtbPS;56%?aY$ML3TjxYRPi%GD-Am%B_2MeK?|&SEeuP>&W!t%R3h)R8V`;D=gkGc zdW_Ou+ynb6TA;&X3)Y>%No6-CpTdLO)oGFkE}6TU^H>zc&E19TYe(3Qi6PeQG`y<< z9h$>5J~Xt>MQOpUk%?_cb9PlC=n$!r@=eP>V#PohBO;{M3L<}BQnm2PHY4Q=v^d5ew%JRw;b ztTv3^VcATMRAl8+F2QC(F#;H>%iO8JGvkp#! z;2{isEal=RSP29@UN}1-`@$GZ7)M2{3`~$^#LARqRw1BMG`Fx>T+-;uV($g!9~G_| z2|`&6*qEmK24S5I<*#1VxH@WuEe263*sC-8hsBJkTTYh$D}sYnov->SDvU0@v{kUW zl<2+W@`eG_V;Id5MQJ^xG>Vo#_QzEHfe%%m-kjZ_=cETikB&ehi31~tAx3k`7JMMf zc6G8~>4qzB!!F2t6e4ieS=uhYJZtYDJsU%Mwh8%(vFr&I|0%F_hT^K_Uk4S$|CWt6 zSc@zpi`%DJJrM?$;{VO9EAfq_UNy_OY+zq)6yT~=2Gn6fnj6+<62YyZL|sdV4)TKe z(uF{T*91pXm7d@)WHkyo`g0Q3<3U3g)kaE0_1PlZG{dt6xig6sb3#R#ox9(|sRIol zUx9R}lh(JRipt2UAHty?3x#lX<&XOusimXqu+Gi1U(UDFQ&f-?kPT4mHBQxi`n0YD zf>Cr=DOhWEa zzG&fdl5G@)>2Tzr2=LIQL?`Dx4#hkGhMhj~P|2q&HG+Gq5k}Ht-Cbv2W)9 zU_hV0bg2|d!fwrwo6`|2NWtVn4V{59wcBGA^I8dTGBSLvz%1)Sv-d&)>5Z~N24p)^ zM)@b~3+Gp#bv)+AzxoU*BYXn1Daq3%zK4->0+%urCC;U~PZ7UmD^e}MIn|TMQ%yBA z09BhQrfg=wfGp$xR;+0P2_RoUB}!mrKei%X{t<Ca_XnhOG+M^iD!P4XJL1)CU zua8mrsXj&&bL)pPr9_MnLKxc`5qn<}uN&yW9lh|B7^US8LxJ1G9YshRQ37Y}Fx7Qs zh5g%T^)S6c2t9uJf#Fb1V9~ggVej1iTY8J8FAmxiIZ!7o-`Tr;SV}A(C>SXbtu>N# z(Mc^u2s)|PmGsRelMKCyOLT!`vC8WDx58a*E2?vklX{t4#{mrA0y<1St*rDZcE8J6 z53v(h?g;v*#xNExyMlub5m0cnkbh2K(zObx`Opx7cR6Hs%q$0KO()iQ#j2B^O2%mJ zjdQ>vCnUn$S%~na@I++aQy6LNPI=-IuFNd!p;fy5!uk8x|B*j@Ubyil;1v~fs~`0D zum5{_T-v`*>7;~RirjIlo$Cjc%uhvvK2U(p-+}RUaqbu%Cu35G?v4qLJINs-4X}0S zlf5c+b}_SVB zkj32KOyKk!wVl>4+ec3O+PS>A$!3TGahp$(5eKs$66JbZ>ug&$D8_=dYeP4Wsi`+K zf452fP_5qCm5yrlMSTikR>N+`zz(V=uwu;Ib8$5)sYH)PQf6H;x?TIKedY8%;q;d; zvo5x-Xoc{*r44vH5NS-_lUY z@&In}VZQdTpZ?Zpu*?!r7DA;K{|GW-`Df^NS{NXZ%OEl;s`}Z8l#Lp=&g6cV4&e=; z_&QfS=HpNFY=;n+R3@%w7JEfQ*9;+O~oAbKbgn({kQ9H^=Wft-F6|pJkE5_TB&8y?)zr zI?nNFtNiWe3eg7IGihfXp*?@;T(+)SdP?HvO)*Zk^-X!(>2}-a0x<8Kb*^a>+?*df zusN=rW7r2eq<-_J8Ed~1`%>}%A76CeT%5x!ww#wOur_dRFJJ~S%C?xt3(z9L?zs$S zaU#-+3d9PZpR`T(gp)L5vTIKeG7l@ZNIfm@T7?s}X?y#hk5Js97Eapub?2n>PdDu{ z>&JlFLm0N@2W$`{NZZ!hZtQNGwyX77DuPV!r=sNT2WY8TQPV<%*9B#2#c^|Gwd~G) zQnjZHDD4SaT~SD9yIGpC^y5mh@~RNt;bX7JT-b*LA0Cj!lCe*sWCszx$!)5_x`bKr zWr#}tp{`+G+LtWT(z}}xU3+{mNMps|E3TKYs@#!|rQ9aWpX=|XtYjHCQf&}80(UA@ zMa6m_Fq4LL^#JYH#OaZ2t5_9U@=OmIQDt*Dyr$$hmt2l9Rl@Tm9$M9-7GS`e|zNAFy;_J*GHHQju+3!zda7Ehzn(v#Ky$S zo7dm4b?(8r^|o{U+G(LKaB(G_dtcr|IVlTLOYOI$Ad1do(Sms$VYiLJYe@A5NBVW< zU)y!8PrCb`kL8JM?2C-+F)wR(V=onuzQ27+WW+u%qDl6S4#HaODNZJ2Q; zRctOz1n2YX_Q_TRi4kP6X|ir7cH95_PP88jiY|W)1{uyjV5x(uNg{FY1q8y@M4^#6 zU0KoRX*qZLD4bxKJ5Gs0pnC}BR_Y!ohVAts_W{;gWkvm94fmXqdB3d9BTR_Yr2xM!G-*C_` zU7yd~Y`Et$!d}z+;M3jzj+Q@ourDQPpvmXDX>$$AoQ7)za~ktCg4vFPkVwYxos80J zL^HP6WFyzcuE`jf=<7=xX&2I72I4~Bf}%w99xXqJvCT&Zl)?I<-oK-XwfeM zDSaT)E5!NxCA6JGOcg;{9)(vIRrOZ+!OB{pb zO+cHCv!kwfzO8<@pVzCAQMv7p?_A0jX zIhz%-twBm~GScCZtW1t=GJe;y0Us83a1pO_9!3-V3<@u(x~S;Ju+EXI5<>=J)Hsai zj&lZWxn&V@_e%)lE9Av8H^q@eo$s<>gysSwM(Hsa_SlgH!H8*rC8N-jUI@^4r|T<| z?^Qb4%E7FQs}2UUe)>z!-=V`T&5&9rhkxfF@iim9z9pc8y=`Az-kiU=2AiP_3&r8h zvMEgiz@kaz!Ff_y(YMT%oefS~mKMT-h=kG>5vz)TV^)Jm`Vdx+%#(`ZN>v_Tt{hLQ3_kpja;w4`Yg}>&2w`$l9I;I*&dZw;bOM@ z;XQHNz`}7sM!86cB@VMOlB9n8ZX6SL6){;&p()`^aK{lt2^?^thjejiwD|a~hfJfy zvCZ8moIU8dX*fGxXN%`m3KC$)|`s__7Byb-!N zsZzE9a&8jR8v;)01GLKt3pe4g5;GEWxQy)`K(+M2R`DaY~&*^jupohFJH%0EDN%FIM z8wwDB)k!{ret;4|cF_uwms<@TK!8 z--HroWkP;dZ&Sk$UP%-qiY}HaWg(FxJSOr4r2i?cpo3(TQd9@hb zD}zujFcq1{N~9_?2W6nhSEJeJq8M_ z>M%Gg7K}f0$C?wjU-EPHDQ69IohTNBi2`2<$K*Ujg+ESG1Xsp83Km9$B6^Fd_^`lm z!DJ*ehDi?nQ5gF1PBk6HPEkzs`7M$G2(%ffQ9}e9xf>8s?N^>Q0<%{{Q)RF`ZCny; zv51jLnU7?Va%W1Ynzj%@TS!4yME==;F|T1$?ac)^BQztx&x`=$oUn0s8Q`bZ(5OE*N zj+^NzRw)1L_6e?Tq7`X^^55J6mxA;W65i*Ipeu!=RA=J8*Ez9y!!R1I(Vk=Cdw;b2 z9t=s*+WYwhjrp-R4>DH;h+9f^;ENn;nk+w{!_7s_efc2?y->QF5sEEKIX*J0Q(1Tr?3xu1BQ zm0xEL}k1q?9?dDdiGI+Z)5eX^QrSmok-jM$FvAAvuj$-<8NZ9@OPL>2z*Av=Bhg;!`38 zR4HD9s7&J%)Ra!*yCut0bl4rBRlz49%$s^%t$}50U<8|k4h-jCwbig!M&hYB8oYl=c_B=lp5A(o?R}S<1 zS!)A?mL1C?Z(gA@K9?`&F|qs$s4u-9(*Y-3Qp!!Snh%yK5`Ca(+(Y%TAf}!_d9r_4 zZEPJpeo>+*&9_3%7IyeWDdRYd3?odl-!sQr28LZkqUMi*NB=mvd*x!Bu( zQ9BoVYv*Fm`Mh(v_p>jYBP6UY6PZdvYnTe&)PZ5W4GPE|iO`uKe*`Sj^sAlhyKFxU z2&|>Pj`ezDVq3s@fJuw8Qkh$ok+`jmQ8-XxGM-lWN!4UjCczaZWB-esh8M;hC!?HX zi2)ce9yW6(+s{cJ!U*m&QV%lbMz0R*dR1l+7KuGrLm~=T@`ck42UwMl!71xTp7_)$ zM)7t`w`KYW!7`pi<)$B)SY6h{OxA{5UMv;Oziro({jS$0&% zVyL=fGZu|S{!p+(UknBIS^%-nvoNomF{FSru9wEhG&elAgY$R0rIdL&W+kun@5?wDj^wEUuD^@afaP!(>O(< zLW>gGmMuf3UgK6U?QYgULn_!bur{SL%O{SxKE;hPTwWaR=Pf|W+1KsX>GkU>uSto| z_9#S>-AHCIe<6!lmS3HHd*3ZtFEivGa%hX0oAEw7aif-2&)05U`&Kb;HE8OFvs&cB z#yM41lbBPiPH4`l{I+iES--x(F`d6(rv8if8*W?mW!=77|MqJ3VUXwV(im5IAo$h) z{`DWeK>s)#vRF{M+>XL8wKJjDDS*aktysNFO%JclCaDv&Eu2#R5L?9b)mgN&O6Il{ zL*@fCvba9#r=+_PUwwhmMD3T=P3eBmra@zHX--eq6kTOwAB zPTfJ=s-Y=rB50ZoRmNG8D_PkgsPwAWmGoO{YtgKfa-w?6Aj2PIH5DwYw^TrkbW|J3 zIs#YJm2vs}q-vYUc3oW`SG8@8C*eh1r1@!aznm%WIEW>XbLFDuJI=z5!(JHT^8NUo zCu^SvJRXN^!l?}5Fm&>A)aG)LRy2*6n>jd3ZbUzv?c(zY{-)4UYZiRNE}}D#gmxOZ zc@lz_mAKy~>_e#@3l|^n#$X8nLRfomGnXwmNG9BJXwd%<{feLDQZb-F+gomicHW@hGS(nS$}wd# zV^L5n7P8w2yj2RU?QyJ_O1UPYL0d!t~O@l9u6 zH9?EV?KExALrG7gmStVpzUuT59RV-*l@@9D=M-Nry8E%oM;H6JVMXQ)ZDvuN6!U7_ zp(@0=3l`66GuO-~v6n4*izp8{U$I5u0t-(Dy(+2K?5C5!}JB zXJgUu*u{p6jVHd&kRuO2EG8X?^?G1qWT-?2r-J?N(kL;acA}jzMlJvo6Ni>P5w&oq z774fg{+gJ$2J3$zj0@k3J}HtNMe-?vsB>Rz!!fDJR5ZC1kO9^eAZJlbK=LJV!Lcd0Yeio0 zl!a4?1l7Z0|JA?nx zn8aT=SNMP4my3p07A2QAY0ryiC1-v2tX;4`s)HK8`s^&c`_bou_zV5@h_69-5FAEe zwe%Ox+1>9h%ME#1b5Y%Cv8Hv^aRlK{<5+a~i##Nxs9Ad~01--bxA3s~&`|yL_Zvzh z_OH^kM0-2M!T@L=4BXo=VvD{@l`doF@=JPdDM!}!w%T~2RqM;Z3CUmfM;dAw4R#$^ zwuw6&a^(ZOJnIls4TGjCS!VK6kW<|5-ieT8W3s_RfjicH|3VDzO)tq zVK9vo;f_hX<5=RCVMWKK^;7>um0{Z(Tpd z9QOvV!c2#!&uU)vqUdC-1avc<;!2n;}4kX9lh%t?N!jZ!?behwVWBk2g?X7QcdFxnj z-l{FxP;lPk#4Gg)rZk&JQ0_p9k`cztHkIqph_hNuX-Q`|tkt3lwNSbvK1lYjF!%Bo zhNis6HBiHaX_N-RL}IsGJ5tMS`(of1>MP2+n8498-WWOi?1|QxI(<= zx9+aLw;s~IvNLvgb}bZ#HrC(2{!Z%binUkZeEWob?N?_Hm!9#+>IwBod8=Nn%e`<) z(QrrWnkp?MLU)p=uV3lXGM@6t>KS!{+^%!$asbL8A}SC_t$cTeOsBRdWd5o)qALT4jqcn~Ff;-1>cMw4sAR2hXB>)MUa9n7+uyg!5_nea zWqno_rI|M?RFss8BTTwkXJO=fJNPp>UDE_h!l;{cx?@D|XlK(EdN0G`3`aIXU z^A3s;0cl$tOaswsT>jK(E6HaqxJBTP zk|1lWp?lPL0Aq~^kV4Ke5(tWXQlDs<+^mSNDA(Iix31*XIVjc-mQ*MNg~$q07X9}D zv@(pw!gyyHwT|*%6DMNMO%lSNvr!1*pg#t?AahUf;5PJyUmiy;n@w?R)%%*9(g!b! zoAI)Jad~Rzdp=Fk0f5xkGcR)QeiXGb;V0zy_9k-$j!@NhX#4s>m?)a!_wkss_s+~& zxMLU;&d7az_nRF@AZ$_{<-L$t=nT}2gd?BtKY8*TRM`C|dryt%)cTgM*x%LzQ#$l7 zVOHt9cTjtF6WGd##*8=0?I~sJ$V8u=$|N5={fpT=I5xYhMy9Umt5k-y!&2w_JtGV> z`hA^68ac-)?0^B*r4qdVu_N+Sf>ORr{ICf#^<%WYA1n36z2Y#vdHK@9k6%(4;VLhL z1ps90p7~hri;i0_VODf;mF{jkgk|+zJl@3toZ^`)>WCw1={@VGiHqOF zD%WqSo-tXOc?jQcCGT+UVAXeKr9H4((r99Aotb^gdLY{-JV}_p5{Yc8>m*{-^IZOY13P|)>hs+ zUH&J@w5q4ndkd`kWM!4!{WzZm6#PO()(=F0j29bY%+cN7EbppEX_UIqZ6Dnr!*^r7 zqYI-+<)e%vQTw4D8nC_@O`LySECCf&Q6Ln4sYd`0)|fg^ySJz8;ZcKQU{0 zPT)s;Uj^5R{!8sLw{OZ?b__Oz*HsW3>p#@PT&&w+wdHzjG38mrH!L^lrSbC$x6a|d z>F>y3iv^_QsXHhAbP&Sv=H_wwfgm*-q?9=|MHDQt&?m$=>g6~@4bW-eIr?GH2X!wc zg|<0q{oeCBcv}gFST>NZ5%`(A`4D51r@F4lO)YEPN#Qvwc4(8v9v5{~ED&34ZZQ{Y zJFPX7tSnor)$zmJ8_iR{D$QoIMjGi2)zs??P0<%$y`D@0I+cl!XEt>Xt=AEIeQ+;F~;Q8-t(MdBv3ealA;}yKnk# zpeK`6WCv_E@Afw{R(%*y&knH-`aLM8jR~U zc0#bpd{}^hMFE2Muyz#R-rSivS2T8Rf8@LggP9n)_lESjYCh2!e}H?fW> z$xbWCsyKReT+5E8z>5?k2OB0Gg+rwpSW`CWx23y(}^?%~x zL?JF|N3-&fraWZ`Wlc)RCDHaukkj@bU#HGfVntncC zyw-!7f&zz>%BbrIRKtsDWo1OnNq|LWZsbvkzzC5nqDot{YRy02dtR3b^ZfCi?&JAt z>W7i2CaHAeCg=6DQaG-CbJe=OINLqxbi194bNx-?7u0&9AfCE@qWnr-Foi}w9V2P7 zD8GIE<@FBU=8Zeyq71z%bvu_NGQ!rRdJZC6k630hOkPou-s~Cm9?|CAIG3*;NRB4T zO10p{X>i{*m&A!AxKl%hfK_~LNN=n>UhHnE4@GR;NP6M&gUdxoDSv0>fZEv$i>INQ~t9eJTMAe?P&C$eLqaS{GNt3(bKv#pBJW0Br|+AW=h!K ztF}7JWk_Ix-bBpf$p92PpZoz5|CDyVfK#7pbjO|>`D`RfHeR_A=J#VdY5^!kKjxW- zW+}j9)`N!&l7clfl9tWKD!+xNin`sBsujQI0Zuu^P_F(9^yrZ+Ivz^6BQyYe!0*(BZuv~DXfcs+;`-TjE6`X3 zwtAE=K(Ib&jI z2v+5>)N@5?T&mZI%8)C^hUX4`aT#$2L-RZTm1nD>PqYM1KU#k9aK+4J$H8=vur261 zK1JPJiXP0xcQ^pjB5EQVirANu=d*p@zIxRZOwgwDqTGh;+j>*b;^o$^BYK?)t;xTx zchjtEMiUPnotiM%~iX*c6U@qeb&(Xthlaz16eoJmM5lZsV1vC>X%#7 z_OA)RG^u#D=GrYR4vScSsb*JE52$T%&BgVbs>yn87_eJYEo;?<^#Zx+5L*P13$LH6 zU~v7aFfLnVAHJxoM_8(TTqHpEbrx{NSR=Ub&;~#aLm)PMk+O_6p|fQCa>I)x*cJRY zx+$shzE`V`jl0$K-P`U!DgAu^`F^!Oalc##T;tM{(hOKYWh(i;Sz`0Xkyf z@*2!afv_K~8?bpIHjQ}9V$$5HInao?*swg7wEdv?hZCv}?4=sh#{+cFWa4kO=v38~ zn2nPVnP0_(&1b>gAHWRmfozFSQoF85+j!+tKVEIeBMB4X;rOpffnA5QUuoU5Ekk>*;Ohq-9&0i zsHGdx%cZcS0GRB&XOcv%kTp$SuH$^g%Q#XqMVJVa55pRiaY;ZN9_-Zxpi0=0?!@(d zr^O?IFD7Ey&%3SbpNYSbmYzAS$fU(trOiLnU@JoHIl5?OYjjF{tGl&?t1GYEw1sv` zx^AfzGrr9#11m4B4?mPb%-!w=V7sEH&xVx@~dBJRXv(n<&)9D044Ft+bC{hfW{bwER;7_ZvNGe0RO;ph>6J zdW+=33509=^IWGx#k^0;Mi5E&X}qcSP2K8i{E!XhSfi}75RtJv&f6@7RWMJzVo2W} z7ZGu$TxfQ4J+#3_F0aoSQ*wzU>@HkN0t-hILjx#ec?H11{*(W23QabdxAz-ZUx@RAnNS2$)jQJq#48tDVv;02r7&4+5 zT@klg+;vZ3vL-WFKs}{MP?`^h7#-4g;)GXtj;-&i2lCjo-4ZaKj(Nsr96)o%qB#qq z{~E~#&GGMYKhB%oyQ!qH3q9z1hT4=ASoLeRZ}pY+`a-cl*Y5RI>%4n;akVj5TW;VuE`~}-sosGNR5c(g zFRqtk7VWh$Gqf4W&4pIh2vf()Zm7ThUW433^!}1XV>XIR*?XD}l7Tyl5~KNcx6}t_ z8Z47_YFsvwaY0r7``6#OaG4lTxU+r+*56kj!?W+lT1HLPoZOlr$Of(C<)3QTz12-C zobpefRTJNq++>%2DyjvG$~VgoI}W;|V}$qwFnGO)iIPE6d11W#Tf~TPDG`6x5j331 zS~@#a)Kt49LWTs0?b1~^)xgnlX27utp6_Y zPnMqwLm-(M@&`Cx1YL_(5C6f7s^LF)>=j};Q}m*H3l{Jcw2TPLw}TmorVEGlkM(x2 zh7lZE6*;$5+ei)niGoLde%U=)bH44ps84M!IAAZ3^NKGrw3Y9qakx6aU)&<=DN8E0 zG$vNIEU{(XdZ?>vQp%JlkmOQaZ|b{=ONrC5oGven$SIgp=vNg~5ECoe^>wy=d>}rr zL7i@^c|T)G5>dSuctaoa-wMD$)Ou6G--_TEOl)N#;09~J3}e;Tf_+wMY9vQ_2kTzD zeVoOGqJ;`@wEb+2%taDI=!+`Zeihe?dLjt5{zyGAmb6S*SJTAH8Pz zSa)psL9_$2&P2yB9H|;2I~He5()9M_|K}^-JkhEOk-gRK-Fx1!cW0oGKxg6}19c=~ zyho&$=g>n1>i#FuneUT$ELve2^xN&15jFmKU0Ahr))rXcM704G`xJ?^vgG70{Iy`Q zS)Mc@kn5kcd@K~5^2~dy6 zci*@k(qB*b8UhkkV5)E$Xk{r^A6r(E^#&v<>t{G(F6wL!1686-2-6CGLjzfAKzJ%k zR2b*g6@xiixP&KzW-R}T3aB>U0zI`7=BKFRZLRnkxT7kJXtF^Ut4FTVj4L@0o|bjC z8*Lr3yZ|bme7QE^he&GxIxWl<^bjqX2<-tuy*VjFGb(x*rcqz3C9N&|WaK8|QMjKI zs<4*DuC~%Yc)fT(8hW^hk$5;Ix7#3Mu*rSmn^RWgL$%(Jrcdh%itd*+YD}?4?bdfP zZV$?{)>0!J;*kYmA7a2A9ac>4CSC+f<~&H23o%xd>!8zbBeJd#w|xgQlR*)? zAn%7}4p6285o^e)jI%I>BPwy(!(H#&P2;q$%o;W-15ld)>LdxxR4PH&+*z?F;9)<8;o? zFMip22ZPjh+RkKDoH+9=*Off7R_?w@*)@`L9m8KXY0)-ERBpb?fvc+|hN; zZ~p(@uBEwc}FCWWh`0t+{uon=&@{zQjtM_y&8U9jcx@RNafsAN$Vx^iXa7aJW-t z=+;vI1M1d1k9rm}nM%kBQUSZ$vYETW1T;^UMG9VGtYU=00l;=vYd{NA!7brn1C7UZ z^a#R)$R*Lk0+1E|NabB(fIv3Vi)FELrQ@ZLz#~N8%zfYiv3-SEtArYnV7UZ}YBx=D z!dGJHB4Oq*rm&>o;mb%L0!+yxj&K+bIj;hd3I7%I+)Q4-q0G>rP{701!1pb}?#>r&T0Bz?_l@b0ZRBtGfX^mQUx5$Et#PlFm=@JK+EM z1WP9F&^+>|zuwEV4a3mRfqWY+LwDsMAjSd8`O<3}SC`4s?V$Y#~ zvZ*7s3eoe)9v+d z2Dg{ai_UrHUH1ciB$O&V*YCXTefj6-AGOR!`N#9Y)w}oI zSBFs5EoMt7@4`Vqasz;*56G#1+eJ(PANJmgnZksxE^h%00yZ36bbEtt2YV3>j2+v@ zBup&0brNp@33a-3GiWB=8-zx3Dx2ST;3N3Ot(Z&?Cj+xVOahO-g4w|3qA?y~I<4X} zJ*+T!S?HT_O^RfmpyMQpH^0P>iPyR%5u}mb|3q<41Uwn*{lJ`?`%wU@VRiLL6&`ffgO!P{;>HeL$8JO z#YJv%H^)dZY}YjXn`-8W276|pKs)I4!Ht4Z5a8~8{y6d1*#SOs=JDq5;Tm?<^^<$1 z=er*YDj7XpZ1V|*%HuA}Z?7*e9Hg-J7{CQrAdtV7KO0)$p3^BkH=_B*rmCm&A7mS^+pE6S*wN%Qog z_Cs98Ynj?|?>eisx~1qhtP68DAEI%t?Sfmwn4#=bRH&K=Oe1-M2(_|32zk%=EQDvM z^e_#9898#h2DVxTC|?GGf|R;3q)Z=aYL{a8unL?AZHuJ-4V^caw>zus)?j0(>BSK|2*PU)#Rz!pvu&LTh^<&SitaW)&p`Y-*8q;|t}=gJZ8;V~I5QiCTM@L|+M#Gha* zN%GH_N!5qI3??(0ZQhymt_W@xnKLG`8|wJRnkAi?>7@P}%;nj^if!fYK;4nXnd3qOTDauoOrn-qu!zfL z!cpU3jw7~lk__Nu9+h!!wTiJCZd<#VUgQB`p>qgPZ)ah3PA!SoabG-fJ!Run|!!Ub6eKfpb;8E`I0WKLcG4ZX#R`^n6rBYTh6iWGug3Fi2Hx^DK^o#SAg1B~C#5j(e)8L8WG+aD#t?Z4q-n zfWvm^tog#+o;@vX^o|zl^P*!XiUl1Ze*Y*{i)%U<_;cR2*wWswNW=kj&A#J2PR5xa0xTjb}IJbR$^<++Rv$=wUVuW?JeEIWo&M*A(9ALVR*5et+ zE~4Ysn^G=~@Kre!dd}Sui<^Ia^)BQ#;R>!*;D+%m#9YUQFR5@aotUq=R;E2BKg6?O zf@JhTAUW%lO9MU(*&PdsJXi41ODZ*h7aSRm5LX=0m5+kEc;yP-vu{K;1zKRoncs;m zMr<-+lW)fB5etu(vo%O(P%4tu=g4s<6Leq`(+_hO9x|qlF`-ZxEE+%~RHg*@UE+@= zUK6tX$eHo!%yA!F?xj!Z@|^~Z5||N#6a1lY>i7}p4p3|BC8ppk0Tc>%{E_-S4uwq= z4^7kUDaD9td-5D|i5K%@ILTuUqwv9G#_@!KOARk*xfgIqvZafJ%MaTzuXQ?VbQU=? zfVbFfM{c3mK}EC#Krp#;aVJTb2e9K-W>)uZKojo}%eI2M->;UrPo2zg+ z12}o!PK;N8N_^LMxow&=#sL7O$ z!$$dVfQVuow`(Ng@e5J7<7dW-euk(GwzPtm(L|d@pVNC427? zVGO@~|Ies(d`_WEk}-(_FbYwSI9O(eSl^8mW+PDSMv3dCKk@~^Ekf|AFQv)yR-DhL zq=&Tey-;v<0V-VWZayi{&wx5DoGb4){vvamNR*hEeMYwEfbDfVH?MlKZnM*ORAEui zxq7KpAUPeq{^zxdt|-`^YmoOVO~`?29lqSMA#+@ zSjGHzOK&Nvp%Sw#HAV7tMkK`pIMHAjF#!8iIW95O9V`mZMEISsRscz1IB&(!y+<%` zv=(QG$Kp7l!$;Z+G6P@!sbp1cObJ%yu0EqgxpG{HGD1lP8hyz4bf!l{aD3Em47<-Z z98iTP=8SgV5FVr;a_^NPt2JJRBpe~Bb;!07{urmd%7;?GMfuZ?*Ls3B?*eIe#p=u$ z2>IEO2GztTMD`wNwliH6BxQa;Y24ICKCJ-CCq*;$TD&$-DmQkU0`tZ>N;~8f9kLr@ zaat`B9c?od_t`|#a8gvEj^{=oLXyD?h(2O5+MP-67zgOXUQt`vM9}MCEPl+?U@_Ji zAUq0WgMTK8M5GbGBbBc65O`Q-J9cL7BWz1!Szr~iej(_&83tpNii^u4s+Da*$ox8F#fzM@u0RrwGr`U#i z7=C-t(gdg`0Iwoa@VuzOE&=l>_l<5-5WA`31=N5XT}9p4KytOr71C6rSZAYKWs~pN zQrfV|ER)JOWtHJRZ#%iRNnVF)zi5b(D*sopxwjJ~BRtKqh$krwE*xLz#?T~7Q*l=1 zAuZf4IH`p4csPbv#z)QroAI$H?TiOBJ`l2cu>2tRgpT6~JDe3*OcH72vQRM9oQ2fs z8GX`1h?6Cv1pCDC0}Mmb8~CV=oBp&iW_Oww*co)2aChnl5l<~B@l(_HRzaVFZ{>z= zd1jU9)Sr_~Fo$~12}b%Fr)f0khS@%?=rCOnSOg9gjH`ZVx$$9fs*#bopBWaj0#DDX0>P)1 zhzst9%)HO6ECKOA|1%#SSt_28qJNz{uXxxZ6Z&eo50u=#|$7D*Nn%nEhkHkIZ0)jzsx0rob| zt)7PuAhAGkT)E#=wkp$|UBrTUJLue8T^_%@9m2wCA<7QtqT>eqQNCEv-@@9vKVMK@ zom$@ZZ5*wL2aJmj-}T|*s#cwBm5LTBMd&e#E@deqWXI$jfuaY0s*jK9lK9TpOE9&f zBoks!-&(1Bu5Q;hN&>nCNwuVgvFKMMi$f)`9YsArNNoNV z;NDRxQ4luB{@ETnY2IdZi+1UVCbY-|Cubj|7@Oq_<<0pQC${UCRqL!Rr`m?!c35_x zhvs>19h|+Nz(J0QJ5Y9g)56$Kj(OJe zLUxaKr(r@U-zRn?p(vTfW!?9e*R_vmky`ZtJC=6mnWv3}jA=C8sRAyuK}JoSGlJHG zGwkFgSJ)VG{iCfhWD~w@Y#R`|i9w96eO^6|<|}n0pS(rFGh<5Dx(6%QXRAxk>YN+RfFtJ(fYzT$$Kxu+D?Xlk z_&dD9nUy9DGxIv^Io;4C3Qb^Qp{^}}=sRR!6N*~$H_NbF} zY5#eb(?aQcppZrhtC!s$wREi8Gc#b%Zx;kYB-zgdrIXnJrC0MdTe-K@|Mtzfkx;FGRlc|C$M+m^^*hnr7Wj~F;9V!ec;Zup>D z52o&nJ^ilq{FlLK8kBSGC2+s}ge*`TOE9Fw1?hrJWa#&TcFg!*6M6P@u` zLF3|;TBH71WJ`1@WlCShd%r$?^5k0weE}yYM$ou0T=~kqEWv`P&L>DBhl{{`pBQ3E zl!;Y9DhG|W8mYVly23H!mq1cNWMp+Y`A9ackc$95oQTDgnIR+IOf121-2!BTrGb>L zL;l{{uLC&tRi_ooF@CC#Ee2^55X;%{Nul$Xa8Pnz$~4 ztCn;P-~2@`%jT2I?G{FjzH~_nMy8#Z(9<+U099O1m0y^pabzhQGFLUtwoDZqHeM2` zxAee}k`89hOBg!CjvJZ(#gN?1hfxc)EhjG>t)kUs?wxh}Nhd=1I~>d8@%hd>P+?RV)K&C!u!q7vye|=$oZjC{477?4hSXmws4hJl1WJ#cs9gi ziojvdeC`p0=fd6m*{AP!_yY9h9JR%Wg^a_k$LW#g6dm&^nX6bGP4#a15$-s5Y`}yn z0uEw{2onQ>V~OK2WMYT3rt%$hW!1`BUl0&=Dh(``EEk6^_w!kD71OPdtOc?7Z Date: Wed, 20 May 2020 10:29:57 +0200 Subject: [PATCH 20/88] tests --- .../dhp/actionmanager/project/PrepareProgrammeTest.java | 2 +- .../dhp/actionmanager/project/SparkUpdateProjectTest.java | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java index 50804f75e..e9755c858 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java @@ -73,7 +73,7 @@ public class PrepareProgrammeTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-programmePath", - getClass().getResource("/eu/dnetlib/dhp/actionmanager/whole_programme.json").getPath(), + getClass().getResource("/eu/dnetlib/dhp/actionmanager/whole_programme.json.gz").getPath(), "-outputPath", workingDir.toString() + "/preparedProgramme" }); diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index d48884842..c41daf2cc 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -9,8 +9,6 @@ import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -72,7 +70,7 @@ public class SparkUpdateProjectTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-programmePath", - getClass().getResource("/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json").getPath(), + getClass().getResource("/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json.gz").getPath(), "-projectPath", getClass().getResource("/eu/dnetlib/dhp/actionmanager/projects_subset.json").getPath(), "-outputPath", @@ -85,7 +83,7 @@ public class SparkUpdateProjectTest { .textFile(workingDir.toString() + "/actionSet") .map(item -> OBJECT_MAPPER.readValue(item, Project.class)); - Assertions.assertEquals(14, tmp.count()); + Assertions.assertEquals(16, tmp.count()); // Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProgramme.class)); // From c0d9e0234050929097896b56a2824a50e262aa9d Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 10:30:25 +0200 Subject: [PATCH 21/88] zipped test resources that are too big --- .../dhp/actionmanager/project/programme.csv | 25 +++++++++++++++++++ .../dhp/actionmanager/projects_subset.json | 16 ++++++++++++ 2 files changed, 41 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv index e69de29bb..6a9c855a0 100644 --- a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/programme.csv @@ -0,0 +1,25 @@ +rcn;code;title;shortTitle;language +664331;H2020-EU.3.3.2.;Un approvisionnement en électricité à faible coût et à faibles émissions de carbone;Low-cost, low-carbon energy supply;fr +664355;H2020-EU.3.3.7.;Absorción por el mercado de la innovación energética - explotación del Programa Energía Inteligente - Europa Europe;Market uptake of energy innovation;es +664323;H2020-EU.3.3.1.;Ridurre il consumo di energia e le emissioni di carbonio grazie all'uso intelligente e sostenibile;Reducing energy consumption and carbon footprint;it +664233;H2020-EU.2.3.2.3.;Wsparcie innowacji rynkowych;Supporting market-driven innovation;pl +664199;H2020-EU.2.1.5.1.;Tecnologías para las fábricas del futuro;Technologies for Factories of the Future;es +664235;H2020-EU.3.;PRIORITÉ «Défis de société»;Societal Challenges;fr +664355;H2020-EU.3.3.7.;"Assorbimento di mercato dell'innovazione energetica - iniziative fondate sul programma ""Energia intelligente - Europa""";Market uptake of energy innovation;it +664355;H2020-EU.3.3.7.;"Markteinführung von Energieinnovationen – Aufbau auf ""Intelligente Energie – Europa";Market uptake of energy innovation;de +664235;H2020-EU.3.;"PRIORIDAD ""Retos de la sociedad""";Societal Challenges;es +664231;H2020-EU.2.3.2.2.;Mejorar la capacidad de innovación de las PYME;Enhancing the innovation capacity of SMEs;es +664223;H2020-EU.2.3.;LIDERAZGO INDUSTRIAL - Innovación en la pequeña y mediana empresa;Innovation in SMEs;es +664323;H2020-EU.3.3.1.;Réduire la consommation d'énergie et l'empreinte carbone en utilisant l'énergie de manière intelligente et durable;Reducing energy consumption and carbon footprint;fr +664323;H2020-EU.3.3.1.;Reducir el consumo de energía y la huella de carbono mediante un uso inteligente y sostenible;Reducing energy consumption and carbon footprint;es +664215;H2020-EU.2.1.6.4.;Beitrag der europäischen Forschung zu internationalen Weltraumpartnerschaften;Research in support of international space partnerships;de +664213;H2020-EU.2.1.6.3.;Permettere lo sfruttamento dei dati spaziali;;it +664213;H2020-EU.2.1.6.3.;Permettre l'exploitation des données spatiales;Enabling exploitation of space data;fr +664231;H2020-EU.2.3.2.2.;Zwiększenie zdolności MŚP pod względem innowacji;Enhancing the innovation capacity of SMEs;pl +664231;H2020-EU.2.3.2.2.;Rafforzare la capacità di innovazione delle PMI;Enhancing the innovation capacity of SMEs;it +664213;H2020-EU.2.1.6.3.;Grundlagen für die Nutzung von Weltraumdaten;Enabling exploitation of space data;de +664211;H2020-EU.2.1.6.2.;Favorecer los avances en las tecnologías espaciales;Enabling advances in space technology;es +664209;H2020-EU.2.1.6.1.;Assurer la compétitivité et l'indépendance de l'Europe et promouvoir l'innovation dans le secteur spatial européen;Competitiveness, non-dependence and innovation;fr +664231;H2020-EU.2.3.2.2.;Renforcement de la capacité d'innovation des PME;Enhancing the innovation capacity of SMEs;fr +664203;H2020-EU.2.1.5.3.;Tecnologías sostenibles, eficientes en su utilización de recursos y de baja emisión de carbono en las industrias de transformación de gran consumo energético;Sustainable, resource-efficient and low-carbon technologies in energy-intensive process industries;es +664103;H2020-EU.1.2.1.;FET Open;FET Open;es \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json index e69de29bb..b8805b2db 100644 --- a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json @@ -0,0 +1,16 @@ +{"rcn":"229267","id":"894593","acronym":"ICARUS","status":"SIGNED","programme":"H2020-EU.3.4.7.","topics":"SESAR-ER4-31-2019","frameworkProgramme":"H2020","title":"INTEGRATED COMMON ALTITUDE REFERENCE SYSTEM FOR U-SPACE","startDate":"2020-05-01","endDate":"2022-07-31","projectUrl":"","objective":"ICARUS project proposes an innovative solution to the challenge of the Common Altitude Reference inside VLL airspaces with the definition of a new U-space service and its validation in a real operational environment. In manned aviation, the methods of determining the altitude of an aircraft are based on pressure altitude difference measurements (e.g. QFE, QNH and FL) referred to a common datum. \nThe UA flights superimpose a new challenge, since a small drone may take off and land almost from everywhere, hence reducing the original significance of QFE settings, introduced on behalf of manned pilots to display on the altimeter the 0-height at touchdown on the local runway. In fact, the possibility for n drones to take off at n different places would generate a series of n different QFE corresponding to different heights of ground pressures referred to the take-off “Home points”. Therefore for a large number drones, new methodologies and procedures shall be put in place. The ICARUS defines a new U-space U3 service tightly coupled with the interface of the existing U-space services (e.g. Tracking, and Flight Planning services). The users of ICARUS service shall be remote pilots competent to fly in BVLOS in the specific category of UAS operations and ultralight GA pilots potentially sharing the same VLL airspace. \nThe ICARUS proposed approach foresees the realization of DTM service embedded in an Application Program Interface (API) that can be queried by UAS pilot/operator (or by drone itself) based on the actual positioning of the UA along its trajectory, computed by the (E)GNSS receiver. The output of the DTM service would provide information on distance from ground/obstacles in combination with the common altitude reference.\nAccuracy, continuity, integrity and availability requirements for GNSS-based altimetry together with accuracy and resolution requirements of the DTM to be provided by ICARUS service are key topics of the study.","totalCost":"1385286,25","ecMaxContribution":"1144587,5","call":"H2020-SESAR-2019-2","fundingScheme":"SESAR-RIA","coordinator":"E-GEOS SPA","coordinatorCountry":"IT","participants":"TOPVIEW SRL;TELESPAZIO SPA;DRONERADAR SP Z O.O.;EUROCONTROL - EUROPEAN ORGANISATION FOR THE SAFETY OF AIR NAVIGATION;EUROUSC ESPANA SL;POLITECNICO DI MILANO;UNIVERSITA DEGLI STUDI DI ROMA LA SAPIENZA","participantCountries":"IT;PL;BE;ES","subjects":""} +{"rcn":"229284","id":"897004","acronym":"ISLand","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Isolation and Segregation Landscape. Archaeology of quarantine in the Indian Ocean World","startDate":"2020-11-01","endDate":"2023-10-31","projectUrl":"","objective":"The proposed research presents an experimental and completely novel investigation within the historical archaeology,\napplied to isolated contexts. The main objective of ISLand is to provide a new way of thinking about human interactions\nwithin colonial empires and bringing colonial studies into dialogue with medical history and the emerging concept of\nhealthscaping. It seeks to do so by studying quarantine facilities in the Indian Ocean World during the long nineteenth\ncentury, a crucial period for the history of European empires in that region and a flashpoint for the conceptualization of\nmodern public health. Quarantine, traditionally viewed as merely a mechanism for the control of disease, will be analyzed as\nthe outward material response to important changes taking place socially, ecologically, and politically at the time.\nThe project is a part of an international, interdisciplinary effort, combining history, archaeology, and anthropology. The\nresearcher will tap numerous archival sources and archaeological data from selected sites, examine them through social and\nspatial analysis, and systematically analyze a test case in Mauritius through the most innovative methods that target\nlandscape and standing archaeology.\nThe broader impacts of ISLand have relevance for current European approaches to the migration crisis, where the threat of\ndisease has been ignited as a potentially debilitating consequence of immigration from extra-European countries. The\ntraining-through-research project at the Stanford University, the top institution where acquiring knowledge and skills in\nhistorical archaeology, will allow the applicant to develop into a position of professional maturity with a specific\ninterdisciplinary set of skills. With the support of the host institutions in EU, the researcher will promote historical archaeology\nin European academy, stimulating new approaches in usual archaeological research and an interdisciplinary approach with\ncultural anthropology.","totalCost":"253052,16","ecMaxContribution":"253052,16","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-GF","coordinator":"UNIVERSITEIT VAN AMSTERDAM","coordinatorCountry":"NL","participants":"","participantCountries":"","subjects":""} +{"rcn":"229281","id":"896300","acronym":"STRETCH","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Smart Textiles for RETrofitting and Monitoring of Cultural Heritage Buildings","startDate":"2020-09-01","endDate":"2022-08-31","projectUrl":"","objective":"This project aims to develop novel techniques using smart multifunctional materials for the combined seismic-plus-energy retrofitting, and Structural Health Monitoring (SHM) of the European cultural heritage buildings (CHB). The need for upgrading the existing old and CHB is becoming increasingly important for the EU countries, due to: (1) their poor structural performance during recent earthquakes (e.g. Italy, Greece) or other natural hazards (e.g. extreme weather conditions) that have resulted in significant economic losses, and loss of human lives; and (2) their low energy performance which increases significantly their energy consumption (buildings are responsible for 40% of EU energy consumption). Moreover, the SHM of the existing buildings is crucial for assessing continuously their structural integrity and thus to provide information for planning cost effective and sustainable maintenance decisions. Since replacing the old buildings with new is not financially feasible, and even it is not allowed for CHB, their lifetime extension requires considering simultaneously both structural and energy retrofitting. It is noted that the annual cost of repair and maintenance of existing European building stock is estimated to be about 50% of the total construction budget, currently standing at more than €300 billion. To achieve cost effectiveness, STRETCH explores a novel approach, which integrates technical textile reinforcement with thermal insulation systems and strain sensors to provide simultaneous structural-plus-energy retrofitting combined with SHM, tailored for masonry cultural heritage building envelopes. The effectiveness of the proposed retrofitting system will be validated experimentally and analytically. Moreover, draft guidelines and recommendations for determining future research on the use of smart composite materials for the concurrent retrofitting (structural-plus-energy) and SHM of the existing cultural heritage buildings envelopes will be proposed.","totalCost":"183473,28","ecMaxContribution":"183473,28","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"JRC -JOINT RESEARCH CENTRE- EUROPEAN COMMISSION","coordinatorCountry":"BE","participants":"","participantCountries":"","subjects":""} +{"rcn":"229265","id":"892890","acronym":"RhythmicPrediction","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Rhythmic prediction in speech perception: are our brain waves in sync with our native language?","startDate":"2021-01-01","endDate":"2022-12-31","projectUrl":"","objective":"Speech has rhythmic properties that widely differ across languages. When we listen to foreign languages, we may perceive them to be more musical, or rather more rap-like than our own. Even if we are unaware of it, the rhythm and melody of language, i.e. prosody, reflects its linguistic structure. On the one hand, prosody emphasizes content words and new information with stress and accents. On the other hand, it is aligned to phrase edges, marking them with boundary tones. Prosody hence helps the listener to focus on important words and to chunk sentences into phrases, and phrases into words. In fact, prosody is even used predictively, for instance to time the onset of the next word, the next piece of new information, or the total remaining length of the utterance, so the listener can seamlessly start their own speaking turn. \nSo, the listener, or rather their brain, is actively predicting when important speech events will happen, using prosody. How prosodic rhythms are exploited to predict speech timing, however, is unclear. No link between prosody and neural predictive processing has yet been empirically made. One hypothesis is that rhythm, such as the alternation of stressed and unstressed syllables, helps listeners time their attention. Similar behavior is best captured by the notion of an internal oscillator which can be set straight by attentional spikes. While neuroscientific evidence for the relation of neural oscillators to speech processing is starting to emerge, no link to the use of prosody nor predictive listening exists, yet. Furthermore, it is still unknown how native language knowledge affects cortical oscillations, and how oscillations are affected by cross-linguistic differences in rhythmic structure. The current project combines the standing knowledge of prosodic typology with the recent advances in neuroscience on cortical oscillations, to investigate the role of internal oscillators on native prosody perception, and active speech prediction.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITE DE GENEVE","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229235","id":"886828","acronym":"ASAP","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Advanced Solutions for Asphalt Pavements","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"The Advanced Solutions for Asphalt Pavements (ASAP) project involves the development of a unique road paving technology which will use a bio-bitumen rejuvenator to rejuvenate aged asphalt bitumen. This technology will help to extend the lifespan of asphalt pavements (roads) and will reduce the environmental and economic impact of roads and road maintenance processes. Recycling and self-healing processes will replace fossil fuel dependent technology. Self-healing will involve rejuvenating aged asphalt bitumen using a bio-rejuvenator developed using microalgae oils (rejuvenating bio-oil). Microalgae has been selected because of its fast growth, versatility and ability to survive within hostile environments, such as wastewater. \n\nASAP will utilise microalgae, cultivated within the wastewater treatment process, as a source of the rejuvenating bio-oil. The solvent (Soxhlet) processes will be used to extract the oil from the microalgae. To ensure the efficiency of the oil extraction process, an ultrasonication process will be used to pre-treat the microalgae. The suitability of rejuvenating bio-oil as a replacement for the bitumen rejuvenator (fossil fuel based) will be ascertained via a series of standard bituminous and accelerated tests. A rejuvenator-binder diffusion numerical model will be developed, based on the Delft Lattice concrete diffusion model, to determine the conditions required for rejuvenation to occur and to ascertain the healing rate of the asphalt binder. These parameters will facilitate the selection and optimisation of the asphalt self-healing systems (specifically the amount of bio-oil rejuvenator and time required) to achieve full rejuvenation. \n\nThis novel approach will benchmark the effectiveness of this intervention against existing asphalt design and maintenance processes and assess feasibility. The ASAP project presents an opportunity to revolutionise road design and maintenance processes and reduce its environmental and financial costs.","totalCost":"187572,48","ecMaxContribution":"187572,48","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"NEDERLANDSE ORGANISATIE VOOR TOEGEPAST NATUURWETENSCHAPPELIJK ONDERZOEK TNO","coordinatorCountry":"NL","participants":"","participantCountries":"","subjects":""} +{"rcn":null,"id":"886776","acronym":null,"status":null,"programme":"H2020-EU.2.1.4.","topics":null,"frameworkProgramme":"H2020","title":"BIO-Based pESTicides production for sustainable agriculture management plan","startDate":"2020-05-01","endDate":"2023-04-30","projectUrl":"","objective":"The BIOBESTicide project will validate and demonstrate the production of an effective and cost-efficient biopesticide. The demonstration will be based on an innovative bio-based value chain starting from the valorisation of sustainable biomasses, i.e. beet pulp and sugar molasses and will exploit the properties of the oomycete Pythium oligandrum strain I-5180 to increase natural plant defenses, to produce an highly effective and eco-friendly biopesticide solution for vine plants protection. \nBIOVITIS, the project coordinator, has developed, at laboratory level (TRL4), an effective method to biocontrol one of the major causes of worldwide vineyards destruction, the Grapevine Trunk Diseases (GTDs). The protection system is based on the oomycete Pythium oligandrum strain I-5180 that, at applied at optimal time and concentration, colonises the root of vines and stimulates the natural plant defences against GTDs, providing a protection that ranges between 40% and 60%. \nBIOBESTicide project will respond to the increasing demands for innovative solutions for crop protection agents, transferring the technology to a DEMO Plant able to produce more than 10 T of a high-quality oomycete-based biopesticide product per year (TRL7). \nThe BIOBESTicide project will validate the efficiency of the formulated product on vineyards of different geographical areas.\nTo assure the safety of products under both health and environmental points of view, a full and complete approval dossier for Pythium oligandrum strain I-5180 will be submitted in all the European countries. \nA Life Cycle Sustainability Assessment (LCSA) will be conducted to assess the environmental, economic and social impacts of the developed products.\nThe adoption of the effective and cost-efficient biopesticide will have significant impacts with a potential ROI of 30 % in just 5 years and a total EBITDA of more than € 6,400,000.","totalCost":"4402772,5","ecMaxContribution":"3069653","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"BIOVITIS","coordinatorCountry":"FR","participants":"MERCIER FRERES SARL;FUNDACION TECNALIA RESEARCH & INNOVATION;LAMBERTI SPA;EURION CONSULTING;CIAOTECH Srl;STOWARZYSZENIE ZACHODNIOPOMORSKI KLASTER CHEMICZNY ZIELONA CHEMIA;NORDZUCKER AG;INSTITUT NATIONAL DE RECHERCHE POUR L'AGRICULTURE, L'ALIMENTATION ET L'ENVIRONNEMENT;INSTITUT FRANCAIS DE LA VIGNE ET DU VIN","participantCountries":"FR;ES;IT;PL;DE","subjects":""} +{"rcn":null,"id":"886776","acronym":null,"status":null,"programme":"H2020-EU.3.2.6.","topics":"BBI-2019-SO3-D4","frameworkProgramme":"H2020","title":"BIO-Based pESTicides production for sustainable agriculture management plan","startDate":"2020-05-01","endDate":"2023-04-30","projectUrl":"","objective":"The BIOBESTicide project will validate and demonstrate the production of an effective and cost-efficient biopesticide. The demonstration will be based on an innovative bio-based value chain starting from the valorisation of sustainable biomasses, i.e. beet pulp and sugar molasses and will exploit the properties of the oomycete Pythium oligandrum strain I-5180 to increase natural plant defenses, to produce an highly effective and eco-friendly biopesticide solution for vine plants protection. \nBIOVITIS, the project coordinator, has developed, at laboratory level (TRL4), an effective method to biocontrol one of the major causes of worldwide vineyards destruction, the Grapevine Trunk Diseases (GTDs). The protection system is based on the oomycete Pythium oligandrum strain I-5180 that, at applied at optimal time and concentration, colonises the root of vines and stimulates the natural plant defences against GTDs, providing a protection that ranges between 40% and 60%. \nBIOBESTicide project will respond to the increasing demands for innovative solutions for crop protection agents, transferring the technology to a DEMO Plant able to produce more than 10 T of a high-quality oomycete-based biopesticide product per year (TRL7). \nThe BIOBESTicide project will validate the efficiency of the formulated product on vineyards of different geographical areas.\nTo assure the safety of products under both health and environmental points of view, a full and complete approval dossier for Pythium oligandrum strain I-5180 will be submitted in all the European countries. \nA Life Cycle Sustainability Assessment (LCSA) will be conducted to assess the environmental, economic and social impacts of the developed products.\nThe adoption of the effective and cost-efficient biopesticide will have significant impacts with a potential ROI of 30 % in just 5 years and a total EBITDA of more than € 6,400,000.","totalCost":"4402772,5","ecMaxContribution":"3069653","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"BIOVITIS","coordinatorCountry":"FR","participants":"MERCIER FRERES SARL;FUNDACION TECNALIA RESEARCH & INNOVATION;LAMBERTI SPA;EURION CONSULTING;CIAOTECH Srl;STOWARZYSZENIE ZACHODNIOPOMORSKI KLASTER CHEMICZNY ZIELONA CHEMIA;NORDZUCKER AG;INSTITUT NATIONAL DE RECHERCHE POUR L'AGRICULTURE, L'ALIMENTATION ET L'ENVIRONNEMENT;INSTITUT FRANCAIS DE LA VIGNE ET DU VIN","participantCountries":"FR;ES;IT;PL;DE","subjects":""} +{"rcn":"229276","id":"895426","acronym":"DisMoBoH","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Dissecting the molecular building principles of locally formed transcriptional hubs","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"Numerous DNA variants have already been identified that modulate inter-individual molecular traits – most prominently gene expression. However, since finding mechanistic interpretations relating genotype to phenotype has proven challenging, the focus has shifted to higher-order regulatory features, i.e. chromatin accessibility, transcription factor (TF) binding and 3D chromatin interactions. This revealed at least two enhancer types: “lead” enhancers in which the presence of genetic variants modulates the activity of entire chromatin domains, and “dependent” ones in which variants induce subtle changes, affecting DNA accessibility, but not transcription. Although cell type-specific TFs are likely important, it remains unclear which sequence features are required to establish such enhancer hierarchies, and under which circumstances genetic variation results in altered enhancer-promoter contacts and differential gene expression. Here, we propose to investigate the molecular mechanisms that link DNA variation to TF binding, chromatin topology, and gene expression response. We will leverage data on enhancer hierarchy and sequence-specific TF binding to identify the sequence signatures that define “lead” enhancers. The results will guide the design of a synthetic locus that serves as an in vivo platform to systematically vary the building blocks of local transcriptional units: i) DNA sequence – including variations in TF binding site affinity and syntax, ii) molecular interactions between TFs, and iii) chromatin conformation. To validate our findings, we will perform optical reconstruction of chromatin architecture for a select number of DNA variants. By simultaneously perturbing co-dependent features, this proposal will provide novel mechanistic insights into the formation of local transcriptional hubs.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-RI","coordinator":"ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229288","id":"898218","acronym":"devUTRs","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Uncovering the roles of 5′UTRs in translational control during early zebrafish development","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"Following fertilisation, metazoan embryos are transcriptionally silent, and embryogenesis is controlled by maternally deposited factors. Developmental progression requires the synthesis of new mRNAs and proteins in a coordinated fashion. Many posttranscriptional mechanisms regulate the fate of maternal mRNAs, but it is less understood how translational control shapes early embryogenesis. In eukaryotes, translation starts at the mRNA 5′ end, consisting of the 5′ cap and 5′ untranslated region (UTR). Protein synthesis is primarily regulated at the translation initiation step by elements within the 5′UTR. However, the role of 5′UTRs in regulating the dynamics of mRNA translation during vertebrate embryogenesis remains unexplored. For example, all vertebrate ribosomal protein (RP) mRNAs harbor a conserved terminal oligopyrimidine tract (TOP) in their 5′UTR. RP levels must be tightly controlled to ensure proper organismal development, but if and how the TOP motif mediates RP mRNA translational regulation during embryogenesis is unclear. Overall, we lack a systematic understanding of the regulatory information contained in 5′UTRs. In this work, I aim to uncover the 5′UTR in vivo rules for mRNA translational regulation during zebrafish embryogenesis. I propose to apply imaging and biochemical approaches to characterise the role of the TOP motif in RP mRNA translational regulation during embryogenesis and identify the trans-acting factor(s) that bind(s) to it (Aim 1). To systematically assess the contribution of 5′UTRs to mRNA translational regulation during zebrafish embryogenesis, I will couple a massively parallel reporter assay of 5′UTRs to polysome profiling (Aim 2). By integrating the translational behaviour of 5′UTR reporters throughout embryogenesis with sequence-based regression models, I anticipate to uncover novel cis-regulatory elements in 5′UTRs with developmental roles.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITAT BASEL","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229261","id":"893787","acronym":"HOLYHOST","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Welfare and Hosting buildings in the “Holy Land” between the 4th and the 7th c. AD","startDate":"2020-10-01","endDate":"2022-09-30","projectUrl":"","objective":"Between the 4th and the 7th century AD, many hospices dedicated to the poor, elderly, strangers and travelers were built in the countryside, along roads, around and inside cities. They were commissioned by the Church, rich pious men and women concerned by the redeem of their sins, as well as emperors who saw this as a guarantee of social stability. Welfare is thus an important phenomena of Late Antiquity, abundantly mentioned by ancient literary sources and inscriptions, particularly in the eastern part of the Empire. However, the buildings that provided shelter and care to the needy have not yet received sufficient attention from archaeologists. Except for buildings which were identified by their inventors as hostels dedicated to pilgrims, they are still invisible in the field. \nThe aim of the HOLYHOST research project is to bring this social history’s main topic on the field of archaeology. It will address the welfare issue through the archaeological and architectural survey and study of Ancient welfare and hosting establishments’ remains, in the Holy Land (Palestine and Jordan) and around. This work will contribute to a better understanding of the practices linked to hospitality, welfare, accommodation and care in Antiquity. Moreover, such establishments served as models for medieval and modern Islamic, Jewish and Christian waqf institutions (religious endowment), and welfare continues to be highly relevant nowadays, through issues still at the heart of contemporary challenges debated in Europe: poverty, social exclusion, migrant crisis, principle of reception and hospitality. This interdisciplinary and diachronic research project will thus offer many new research perspectives, in terms of history of architecture, evolution of care practices, social and political regulations.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITE PARIS I PANTHEON-SORBONNE","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229282","id":"896189","acronym":"MICADO","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Microbial contribution to continental wetland carbon budget","startDate":"2021-01-04","endDate":"2023-01-03","projectUrl":"","objective":"Continental wetlands are major carbon dioxide sinks but the second largest source of methane. Monitoring of wetland methane emissions revealed large inter-site variability that is hard to explain in the framework of current biogeochemical theories. Methane production in wetlands is an anaerobic microbial driven process involving a complex set of microbial metabolisms depending on the availability of (i) energy (via the presence of specific redox couples), (ii) organic substrates and (iii) specific microbial communities. To understand the complexity of microbial drivers on wetland methane emissions and quantify their contribution, the MICADO project will set up a multidisciplinary approach linking isotope organic geochemistry and environmental microbiology to assess microbial functioning in situ. As an organic geochemist I have developed an innovative approach to trace in situ microbial activity via compound specific carbon isotope analysis of microbe macromolecules and organic metabolites. The host institution is a leader in France in environmental microbiology and biogeochemistry developing high-throughput metagenomics and microbial rate assessments, for which I will be trained during the MICADO project. These techniques are highly complementary and combined they will provide a comprehensive knowledge on microbial metabolisms involved in organic matter degradation encompassing their complexity and interactions. This will revisit the relationships between organic substrate availability and microbial communities and will contribute at estimating the impact of microbial activity on wetland methane emissions. This project will give me the opportunity to acquire fundamental knowledge and to develop original lines of research that will consolidate my position as an independent scientist in biogeochemistry.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"CENTRE NATIONAL DE LA RECHERCHE SCIENTIFIQUE CNRS","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229249","id":"891624","acronym":"CuTAN","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Copper-Catalyzed Multicomponent Reactions in Tandem Processes for Target Molecule Synthesis","startDate":"2021-02-01","endDate":"2023-01-31","projectUrl":"","objective":"The invention of processes that can form several bonds, stereocentres and rings in a single process is key to a sustainable future in synthetic chemistry. Multicomponent reactions and tandem procedures are two strategies that enable the rapid build-up of molecular complexity from simple reagents. By combining these two strategies into a single procedure, the diversity, complexity and value of products can be further enhanced along with the efficiency and economy of their construction. In this project, Dr Satpathi will develop novel copper-catalyzed multicomponent couplings of unsaturated hydrocarbons (e.g. allenes, enynes) with imines and boron reagents. These procedures will provide high-value amine products with universally high regio-, diastero- and enantiocontrol. The products will bear a variety of synthetic handles, for example, amino, alkynyl/alkenyl, and boryl groups, thus the products are primed for subsequent transformation. Dr Satpathi will exploit this functionality in tandem intramolecular couplings (e.g. intramolecular Suzuki/Buchwald-Hartwig reactions) to provide core cyclic structures of drug molecules and natural products. Thus, through a tandem procedure of; 1) copper-catalyzed borofunctionalization, and; 2) subsequent transition-metal catalyzed cyclization, he will gain efficient access to highly sought-after complex molecules. Overall, the process will provide high-value, chiral, cyclic motifs from abundant, achiral, linear substrates. Finally, Dr Satpathi has identified the phthalide-isoquinoline family of alkaloids as target molecules to display the power of his tandem methodology. Dr Satpathi has devised a novel route, which begins with our tandem multifunctionalization/cyclization reaction, to provide a range of these important alkaloids. The chosen alkaloids are of particular interest as they display a range of bioactivities – for example as natural products, receptor antagonists and on-market drugs.","totalCost":"212933,76","ecMaxContribution":"212933,76","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"THE UNIVERSITY OF MANCHESTER","coordinatorCountry":"UK","participants":"","participantCountries":"","subjects":""} +{"rcn":"229239","id":"887259","acronym":"ALEHOOP","status":"SIGNED","programme":"H2020-EU.2.1.4.","topics":"BBI-2019-SO3-D3","frameworkProgramme":"H2020","title":"Biorefineries for the valorisation of macroalgal residual biomass and legume processing by-products to obtain new protein value chains for high-value food and feed applications","startDate":"2020-06-01","endDate":"2024-05-31","projectUrl":"","objective":"ALEHOOP provides the demonstration at pilot scale of both sustainable macroalgae and legume-based biorefineries for the recovery of low-cost dietary proteins from alga-based and plant residual biomass and their validation to meet market requirements of consumers and industry in the food and feed sectors. In these sectors, consumers are demanding affordable functional natural proteins from alternative sources and industry is demanding low-cost bio-based protein formulations with better performance and higher sustainability. \nCurrent protein demand for the 7.3 billion inhabitants of the world is approximately 202 Mt. Due to the rise in meat consumption more proteins are therefore required for animal feeding. To satisfy the current protein demand, Europe imports over 30 Mt of soy from the Americas each year mainly for animal feeding, entailing 95% dependency of EU on imported soy. Current sources of proteins are becoming unsustainable from an economic and environmental perspective for Europe resulting in concerns for sustainability and food security and leading to search for new alternative proteins. \nALEHOOP addresses the obtaining of proteins from green macroalgal blooms, brown seaweed by-products from algae processors and legume processing by-products (peas, lupines, beans and lentils) as alternative protein sources for animal feeding (case of green seaweed) and food applications (case of brown seaweed and legume by-products), since they are low cost and under-exploited biomass that do not compete with traditional food crops for space and resources. This will reduce EU´s dependency on protein imports and contribute to our raw material security. The new proteins will be validated in foods for elderly, sporty and overweight people, vegetarians and healthy consumers as well as for animal feed creating cross-sectorial interconnection between these value chains and supporting the projected business plan.","totalCost":"6718370","ecMaxContribution":"5140274,41","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"CONTACTICA S.L.","coordinatorCountry":"ES","participants":"CENTIV GMBH;ALGINOR ASA;FUNDACION TECNALIA RESEARCH & INNOVATION;INDUKERN,S.A.;ASOCIACION NACIONAL DE FABRICANTES DE CONSERVAS DE PESCADOS Y MARISCOS-CENTRO TECNICO NACIONAL DE CONSERVACION DE PRODUCTOS DE LA PESCA;BIOZOON GMBH;EIGEN VERMOGEN VAN HET INSTITUUT VOOR LANDBOUW- EN VISSERIJONDERZOEK;BIOSURYA SL;VYZKUMNY USTAV VETERINARNIHO LEKARSTVI;NUTRITION SCIENCES;TECHNOLOGICAL UNIVERSITY DUBLIN;GARLAN, S.COOP.;ISANATUR SPAIN SL;UNIVERSIDAD DE VIGO;UNIVERSIDAD DE CADIZ","participantCountries":"DE;NO;ES;BE;CZ;IE","subjects":""} +{"rcn":"229239","id":"887259","acronym":"ALEHOOP","status":"SIGNED","programme":"H2020-EU.3.2.6.","topics":"BBI-2019-SO3-D3","frameworkProgramme":"H2020","title":"Biorefineries for the valorisation of macroalgal residual biomass and legume processing by-products to obtain new protein value chains for high-value food and feed applications","startDate":"2020-06-01","endDate":"2024-05-31","projectUrl":"","objective":"ALEHOOP provides the demonstration at pilot scale of both sustainable macroalgae and legume-based biorefineries for the recovery of low-cost dietary proteins from alga-based and plant residual biomass and their validation to meet market requirements of consumers and industry in the food and feed sectors. In these sectors, consumers are demanding affordable functional natural proteins from alternative sources and industry is demanding low-cost bio-based protein formulations with better performance and higher sustainability. \nCurrent protein demand for the 7.3 billion inhabitants of the world is approximately 202 Mt. Due to the rise in meat consumption more proteins are therefore required for animal feeding. To satisfy the current protein demand, Europe imports over 30 Mt of soy from the Americas each year mainly for animal feeding, entailing 95% dependency of EU on imported soy. Current sources of proteins are becoming unsustainable from an economic and environmental perspective for Europe resulting in concerns for sustainability and food security and leading to search for new alternative proteins. \nALEHOOP addresses the obtaining of proteins from green macroalgal blooms, brown seaweed by-products from algae processors and legume processing by-products (peas, lupines, beans and lentils) as alternative protein sources for animal feeding (case of green seaweed) and food applications (case of brown seaweed and legume by-products), since they are low cost and under-exploited biomass that do not compete with traditional food crops for space and resources. This will reduce EU´s dependency on protein imports and contribute to our raw material security. The new proteins will be validated in foods for elderly, sporty and overweight people, vegetarians and healthy consumers as well as for animal feed creating cross-sectorial interconnection between these value chains and supporting the projected business plan.","totalCost":"6718370","ecMaxContribution":"5140274,41","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"CONTACTICA S.L.","coordinatorCountry":"ES","participants":"CENTIV GMBH;ALGINOR ASA;FUNDACION TECNALIA RESEARCH & INNOVATION;INDUKERN,S.A.;ASOCIACION NACIONAL DE FABRICANTES DE CONSERVAS DE PESCADOS Y MARISCOS-CENTRO TECNICO NACIONAL DE CONSERVACION DE PRODUCTOS DE LA PESCA;BIOZOON GMBH;EIGEN VERMOGEN VAN HET INSTITUUT VOOR LANDBOUW- EN VISSERIJONDERZOEK;BIOSURYA SL;VYZKUMNY USTAV VETERINARNIHO LEKARSTVI;NUTRITION SCIENCES;TECHNOLOGICAL UNIVERSITY DUBLIN;GARLAN, S.COOP.;ISANATUR SPAIN SL;UNIVERSIDAD DE VIGO;UNIVERSIDAD DE CADIZ","participantCountries":"DE;NO;ES;BE;CZ;IE","subjects":""} +{"rcn":"229258","id":"892834","acronym":"DENVPOC","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"qPCR Microfluidics point-of-care platform for dengue diagnosis","startDate":"2020-05-18","endDate":"2022-05-17","projectUrl":"","objective":"As a result of Global climate change and fast urbanization, global outbreaks of Dengue (DENV)/ Zika(ZIKV)/Chikungunya(CHIKV) virus have the potential to occur. The most common pathway of these infections in humans is through the female Aedes mosquito vector. DENV is an exanthematous febrile disease with varied clinical manifestations and progressions . Due to similarities in symptoms between DENV and ZIKV and CHIKV, it is difficult to make a differential diagnosis, impeding appropriate, timely medical intervention. Furthermore, cross-reactivity with ZIKV, which was recently related to microcephaly, is a serious issue. In 2016, in Brazil alone, there were 4180 microcephaly cases reported instead of 163 cases, more in line with yearly expected projections , , Thus, the sooner an accurate diagnostic which differentiates DENV from the other manifestations is critical; most especially at the early stages of the infection, to have a reliable diagnosis in pregnant women. In 2016, the OMS emergency committee declared that the outbreaks and the potentially resultant neurological disorders in Brazil were an important international state of emergency in public health, as a result of the associated secondary effects; these diseases became a Global concern. This project allows developing a highly and fast Multiplex qPCR POC platform by using FASTGENE technology with a minimal amount of patient serotype. It would reduce the time of analysis (30 to 90’ for a standard) and costs. Additionally, the sample preprocessing and thermalization will shorten real-time PCR amplification time and will be integrated within the microfluidic systems. This platform can result in a commercialized product whereupon a main market target would be pregnant women and people living or traveling through/from outbreak risk areas.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-SE","coordinator":"BFORCURE","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229280","id":"895716","acronym":"DoMiCoP","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"The Diffusion of Migration Control Practice. Actors, Processes and Effects.","startDate":"2021-03-01","endDate":"2023-02-28","projectUrl":"","objective":"DoMiCoP develops new understandings and perspectives to study migration control in practice in the European Union by asking one main question: how and why do communities of practice develop and diffuse the knowledge required to put migration control into action? Unlike the nexus between expert knowledge, epistemic communities and policy formulation, the nexus between everyday knowledge, communities of practice and policy implementation has not yet received systematic scholarly attention. My project bridges that gap by focusing on intermediate arenas in which communities of practice take shape most notably the meetings and trainings that gather state and non-state actors involved in putting asylum, detention and removal into practice. By building on field-based methodologies (interviews and participant observations), DoMiCoP sheds ethnographic light on the role that ‘learning from abroad’ plays in the implementation of migration control in the EU. My project’s aim is threefold: 1) Identifying arenas at intermediate levels in which communities of practice take shape; 2) Analysing the communities of practice by focusing on the configurations of actors and organizations involved, the motivations underlying their involvement, the process of knowledge development in interaction, the conflicts and negotiations; 3) Revealing the role of non-state organizations (private for profit and not-for-profit). From a theoretical point of view, this project goes beyond the classical view of the implementation as a test to assess the effectiveness of policy transfers towards an analysis of policy transfer at that level of policy-making. From an empirical point of view, the project expands knowledge about less-studied venues of policy-making and provides original thick descriptions. From a methodological point of view, the project engages with qualitative methods for the study of policy diffusion and aims at responding to their main challenges through participant observation.","totalCost":"163673,28","ecMaxContribution":"163673,28","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"EUROPEAN UNIVERSITY INSTITUTE","coordinatorCountry":"IT","participants":"","participantCountries":"","subjects":""} \ No newline at end of file From 3c0eb12d3ea6ea7cb0cd074391e0e5a1d5521658 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 10:31:05 +0200 Subject: [PATCH 22/88] removed the not zipped files --- .../eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json | 0 .../resources/eu/dnetlib/dhp/actionmanager/whole_programme.json | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json delete mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json deleted file mode 100644 index e69de29bb..000000000 From 67ba4fde57d0bd6763c46cc3aaf09af5c51405e3 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 13:53:08 +0200 Subject: [PATCH 23/88] added test for prepare projects step --- .../project/PrepareProjectTest.java | 94 ++++++++++++++++++ .../preparedProgramme_whole.json.gz | Bin .../prepared_projects.json} | 0 .../project/projects_subset.json | 0 .../{ => project}/whole_programme.json.gz | Bin 5 files changed, 94 insertions(+) create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java rename dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/{ => project}/preparedProgramme_whole.json.gz (100%) rename dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/{projects_subset.json => project/prepared_projects.json} (100%) create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json rename dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/{ => project}/whole_programme.json.gz (100%) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java new file mode 100644 index 000000000..fa9e32fd5 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java @@ -0,0 +1,94 @@ + +package eu.dnetlib.dhp.actionmanager.project; + +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class PrepareProjectTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final ClassLoader cl = PrepareProjectTest.class + .getClassLoader(); + + private static SparkSession spark; + + private static Path workingDir; + private static final Logger log = LoggerFactory + .getLogger(PrepareProjectTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(PrepareProjectTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(PrepareProjectTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(PrepareProjectTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + public void numberDistinctProgrammeTest() throws Exception { + PrepareProjects + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-projectPath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/projects_subset.json").getPath(), + "-outputPath", + workingDir.toString() + "/preparedProjects" + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/preparedProjects") + .map(item -> OBJECT_MAPPER.readValue(item, CSVProject.class)); + + Assertions.assertEquals(20, tmp.count()); + + Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProject.class)); + + Assertions.assertEquals(0, verificationDataset.filter("length(id) = 0").count()); + Assertions.assertEquals(0, verificationDataset.filter("length(programme) = 0").count()); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz similarity index 100% rename from dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json.gz rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json similarity index 100% rename from dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/projects_subset.json rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/whole_programme.json.gz similarity index 100% rename from dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/whole_programme.json.gz rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/whole_programme.json.gz From 9079bc1f6125bc687a4d144093c7125651e9176c Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 13:53:32 +0200 Subject: [PATCH 24/88] - --- .../dhp/actionmanager/project/PrepareProgrammeTest.java | 2 +- .../dhp/actionmanager/project/SparkUpdateProjectTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java index e9755c858..7f890a8a3 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgrammeTest.java @@ -73,7 +73,7 @@ public class PrepareProgrammeTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-programmePath", - getClass().getResource("/eu/dnetlib/dhp/actionmanager/whole_programme.json.gz").getPath(), + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/whole_programme.json.gz").getPath(), "-outputPath", workingDir.toString() + "/preparedProgramme" }); diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index c41daf2cc..3e559228f 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -70,9 +70,9 @@ public class SparkUpdateProjectTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-programmePath", - getClass().getResource("/eu/dnetlib/dhp/actionmanager/preparedProgramme_whole.json.gz").getPath(), + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz").getPath(), "-projectPath", - getClass().getResource("/eu/dnetlib/dhp/actionmanager/projects_subset.json").getPath(), + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(), "-outputPath", workingDir.toString() + "/actionSet" }); From 055eec5a77c52d5f4e1b6ec9788fa9645c800220 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 20 May 2020 13:54:10 +0200 Subject: [PATCH 25/88] added resource for prepare project test --- .../actionmanager/project/projects_subset.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json index e69de29bb..edf83fbc8 100644 --- a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/projects_subset.json @@ -0,0 +1,16 @@ +{"rcn":"229267","id":"894593","acronym":"ICARUS","status":"SIGNED","programme":"H2020-EU.3.4.7.","topics":"SESAR-ER4-31-2019","frameworkProgramme":"H2020","title":"INTEGRATED COMMON ALTITUDE REFERENCE SYSTEM FOR U-SPACE","startDate":"2020-05-01","endDate":"2022-07-31","projectUrl":"","objective":"ICARUS project proposes an innovative solution to the challenge of the Common Altitude Reference inside VLL airspaces with the definition of a new U-space service and its validation in a real operational environment. In manned aviation, the methods of determining the altitude of an aircraft are based on pressure altitude difference measurements (e.g. QFE, QNH and FL) referred to a common datum. \nThe UA flights superimpose a new challenge, since a small drone may take off and land almost from everywhere, hence reducing the original significance of QFE settings, introduced on behalf of manned pilots to display on the altimeter the 0-height at touchdown on the local runway. In fact, the possibility for n drones to take off at n different places would generate a series of n different QFE corresponding to different heights of ground pressures referred to the take-off “Home points”. Therefore for a large number drones, new methodologies and procedures shall be put in place. The ICARUS defines a new U-space U3 service tightly coupled with the interface of the existing U-space services (e.g. Tracking, and Flight Planning services). The users of ICARUS service shall be remote pilots competent to fly in BVLOS in the specific category of UAS operations and ultralight GA pilots potentially sharing the same VLL airspace. \nThe ICARUS proposed approach foresees the realization of DTM service embedded in an Application Program Interface (API) that can be queried by UAS pilot/operator (or by drone itself) based on the actual positioning of the UA along its trajectory, computed by the (E)GNSS receiver. The output of the DTM service would provide information on distance from ground/obstacles in combination with the common altitude reference.\nAccuracy, continuity, integrity and availability requirements for GNSS-based altimetry together with accuracy and resolution requirements of the DTM to be provided by ICARUS service are key topics of the study.","totalCost":"1385286,25","ecMaxContribution":"1144587,5","call":"H2020-SESAR-2019-2","fundingScheme":"SESAR-RIA","coordinator":"E-GEOS SPA","coordinatorCountry":"IT","participants":"TOPVIEW SRL;TELESPAZIO SPA;DRONERADAR SP Z O.O.;EUROCONTROL - EUROPEAN ORGANISATION FOR THE SAFETY OF AIR NAVIGATION;EUROUSC ESPANA SL;POLITECNICO DI MILANO;UNIVERSITA DEGLI STUDI DI ROMA LA SAPIENZA","participantCountries":"IT;PL;BE;ES","subjects":""} +{"rcn":"229284","id":"897004","acronym":"ISLand","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Isolation and Segregation Landscape. Archaeology of quarantine in the Indian Ocean World","startDate":"2020-11-01","endDate":"2023-10-31","projectUrl":"","objective":"The proposed research presents an experimental and completely novel investigation within the historical archaeology,\napplied to isolated contexts. The main objective of ISLand is to provide a new way of thinking about human interactions\nwithin colonial empires and bringing colonial studies into dialogue with medical history and the emerging concept of\nhealthscaping. It seeks to do so by studying quarantine facilities in the Indian Ocean World during the long nineteenth\ncentury, a crucial period for the history of European empires in that region and a flashpoint for the conceptualization of\nmodern public health. Quarantine, traditionally viewed as merely a mechanism for the control of disease, will be analyzed as\nthe outward material response to important changes taking place socially, ecologically, and politically at the time.\nThe project is a part of an international, interdisciplinary effort, combining history, archaeology, and anthropology. The\nresearcher will tap numerous archival sources and archaeological data from selected sites, examine them through social and\nspatial analysis, and systematically analyze a test case in Mauritius through the most innovative methods that target\nlandscape and standing archaeology.\nThe broader impacts of ISLand have relevance for current European approaches to the migration crisis, where the threat of\ndisease has been ignited as a potentially debilitating consequence of immigration from extra-European countries. The\ntraining-through-research project at the Stanford University, the top institution where acquiring knowledge and skills in\nhistorical archaeology, will allow the applicant to develop into a position of professional maturity with a specific\ninterdisciplinary set of skills. With the support of the host institutions in EU, the researcher will promote historical archaeology\nin European academy, stimulating new approaches in usual archaeological research and an interdisciplinary approach with\ncultural anthropology.","totalCost":"253052,16","ecMaxContribution":"253052,16","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-GF","coordinator":"UNIVERSITEIT VAN AMSTERDAM","coordinatorCountry":"NL","participants":"","participantCountries":"","subjects":""} +{"rcn":"229281","id":"896300","acronym":"STRETCH","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Smart Textiles for RETrofitting and Monitoring of Cultural Heritage Buildings","startDate":"2020-09-01","endDate":"2022-08-31","projectUrl":"","objective":"This project aims to develop novel techniques using smart multifunctional materials for the combined seismic-plus-energy retrofitting, and Structural Health Monitoring (SHM) of the European cultural heritage buildings (CHB). The need for upgrading the existing old and CHB is becoming increasingly important for the EU countries, due to: (1) their poor structural performance during recent earthquakes (e.g. Italy, Greece) or other natural hazards (e.g. extreme weather conditions) that have resulted in significant economic losses, and loss of human lives; and (2) their low energy performance which increases significantly their energy consumption (buildings are responsible for 40% of EU energy consumption). Moreover, the SHM of the existing buildings is crucial for assessing continuously their structural integrity and thus to provide information for planning cost effective and sustainable maintenance decisions. Since replacing the old buildings with new is not financially feasible, and even it is not allowed for CHB, their lifetime extension requires considering simultaneously both structural and energy retrofitting. It is noted that the annual cost of repair and maintenance of existing European building stock is estimated to be about 50% of the total construction budget, currently standing at more than €300 billion. To achieve cost effectiveness, STRETCH explores a novel approach, which integrates technical textile reinforcement with thermal insulation systems and strain sensors to provide simultaneous structural-plus-energy retrofitting combined with SHM, tailored for masonry cultural heritage building envelopes. The effectiveness of the proposed retrofitting system will be validated experimentally and analytically. Moreover, draft guidelines and recommendations for determining future research on the use of smart composite materials for the concurrent retrofitting (structural-plus-energy) and SHM of the existing cultural heritage buildings envelopes will be proposed.","totalCost":"183473,28","ecMaxContribution":"183473,28","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"JRC -JOINT RESEARCH CENTRE- EUROPEAN COMMISSION","coordinatorCountry":"BE","participants":"","participantCountries":"","subjects":""} +{"rcn":"229265","id":"892890","acronym":"RhythmicPrediction","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Rhythmic prediction in speech perception: are our brain waves in sync with our native language?","startDate":"2021-01-01","endDate":"2022-12-31","projectUrl":"","objective":"Speech has rhythmic properties that widely differ across languages. When we listen to foreign languages, we may perceive them to be more musical, or rather more rap-like than our own. Even if we are unaware of it, the rhythm and melody of language, i.e. prosody, reflects its linguistic structure. On the one hand, prosody emphasizes content words and new information with stress and accents. On the other hand, it is aligned to phrase edges, marking them with boundary tones. Prosody hence helps the listener to focus on important words and to chunk sentences into phrases, and phrases into words. In fact, prosody is even used predictively, for instance to time the onset of the next word, the next piece of new information, or the total remaining length of the utterance, so the listener can seamlessly start their own speaking turn. \nSo, the listener, or rather their brain, is actively predicting when important speech events will happen, using prosody. How prosodic rhythms are exploited to predict speech timing, however, is unclear. No link between prosody and neural predictive processing has yet been empirically made. One hypothesis is that rhythm, such as the alternation of stressed and unstressed syllables, helps listeners time their attention. Similar behavior is best captured by the notion of an internal oscillator which can be set straight by attentional spikes. While neuroscientific evidence for the relation of neural oscillators to speech processing is starting to emerge, no link to the use of prosody nor predictive listening exists, yet. Furthermore, it is still unknown how native language knowledge affects cortical oscillations, and how oscillations are affected by cross-linguistic differences in rhythmic structure. The current project combines the standing knowledge of prosodic typology with the recent advances in neuroscience on cortical oscillations, to investigate the role of internal oscillators on native prosody perception, and active speech prediction.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITE DE GENEVE","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229235","id":"886828","acronym":"ASAP","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Advanced Solutions for Asphalt Pavements","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"The Advanced Solutions for Asphalt Pavements (ASAP) project involves the development of a unique road paving technology which will use a bio-bitumen rejuvenator to rejuvenate aged asphalt bitumen. This technology will help to extend the lifespan of asphalt pavements (roads) and will reduce the environmental and economic impact of roads and road maintenance processes. Recycling and self-healing processes will replace fossil fuel dependent technology. Self-healing will involve rejuvenating aged asphalt bitumen using a bio-rejuvenator developed using microalgae oils (rejuvenating bio-oil). Microalgae has been selected because of its fast growth, versatility and ability to survive within hostile environments, such as wastewater. \n\nASAP will utilise microalgae, cultivated within the wastewater treatment process, as a source of the rejuvenating bio-oil. The solvent (Soxhlet) processes will be used to extract the oil from the microalgae. To ensure the efficiency of the oil extraction process, an ultrasonication process will be used to pre-treat the microalgae. The suitability of rejuvenating bio-oil as a replacement for the bitumen rejuvenator (fossil fuel based) will be ascertained via a series of standard bituminous and accelerated tests. A rejuvenator-binder diffusion numerical model will be developed, based on the Delft Lattice concrete diffusion model, to determine the conditions required for rejuvenation to occur and to ascertain the healing rate of the asphalt binder. These parameters will facilitate the selection and optimisation of the asphalt self-healing systems (specifically the amount of bio-oil rejuvenator and time required) to achieve full rejuvenation. \n\nThis novel approach will benchmark the effectiveness of this intervention against existing asphalt design and maintenance processes and assess feasibility. The ASAP project presents an opportunity to revolutionise road design and maintenance processes and reduce its environmental and financial costs.","totalCost":"187572,48","ecMaxContribution":"187572,48","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"NEDERLANDSE ORGANISATIE VOOR TOEGEPAST NATUURWETENSCHAPPELIJK ONDERZOEK TNO","coordinatorCountry":"NL","participants":"","participantCountries":"","subjects":""} +{"rcn":"229236","id":"886776","acronym":"BIOBESTicide","status":"SIGNED","programme":"H2020-EU.2.1.4.;H2020-EU.3.2.6.","topics":"BBI-2019-SO3-D4","frameworkProgramme":"H2020","title":"BIO-Based pESTicides production for sustainable agriculture management plan","startDate":"2020-05-01","endDate":"2023-04-30","projectUrl":"","objective":"The BIOBESTicide project will validate and demonstrate the production of an effective and cost-efficient biopesticide. The demonstration will be based on an innovative bio-based value chain starting from the valorisation of sustainable biomasses, i.e. beet pulp and sugar molasses and will exploit the properties of the oomycete Pythium oligandrum strain I-5180 to increase natural plant defenses, to produce an highly effective and eco-friendly biopesticide solution for vine plants protection. \nBIOVITIS, the project coordinator, has developed, at laboratory level (TRL4), an effective method to biocontrol one of the major causes of worldwide vineyards destruction, the Grapevine Trunk Diseases (GTDs). The protection system is based on the oomycete Pythium oligandrum strain I-5180 that, at applied at optimal time and concentration, colonises the root of vines and stimulates the natural plant defences against GTDs, providing a protection that ranges between 40% and 60%. \nBIOBESTicide project will respond to the increasing demands for innovative solutions for crop protection agents, transferring the technology to a DEMO Plant able to produce more than 10 T of a high-quality oomycete-based biopesticide product per year (TRL7). \nThe BIOBESTicide project will validate the efficiency of the formulated product on vineyards of different geographical areas.\nTo assure the safety of products under both health and environmental points of view, a full and complete approval dossier for Pythium oligandrum strain I-5180 will be submitted in all the European countries. \nA Life Cycle Sustainability Assessment (LCSA) will be conducted to assess the environmental, economic and social impacts of the developed products.\nThe adoption of the effective and cost-efficient biopesticide will have significant impacts with a potential ROI of 30 % in just 5 years and a total EBITDA of more than € 6,400,000.","totalCost":"4402772,5","ecMaxContribution":"3069653","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"BIOVITIS","coordinatorCountry":"FR","participants":"MERCIER FRERES SARL;FUNDACION TECNALIA RESEARCH & INNOVATION;LAMBERTI SPA;EURION CONSULTING;CIAOTECH Srl;STOWARZYSZENIE ZACHODNIOPOMORSKI KLASTER CHEMICZNY ZIELONA CHEMIA;NORDZUCKER AG;INSTITUT NATIONAL DE RECHERCHE POUR L'AGRICULTURE, L'ALIMENTATION ET L'ENVIRONNEMENT;INSTITUT FRANCAIS DE LA VIGNE ET DU VIN","participantCountries":"FR;ES;IT;PL;DE","subjects":""} +{"rcn":"229276","id":"895426","acronym":"DisMoBoH","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Dissecting the molecular building principles of locally formed transcriptional hubs","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"Numerous DNA variants have already been identified that modulate inter-individual molecular traits – most prominently gene expression. However, since finding mechanistic interpretations relating genotype to phenotype has proven challenging, the focus has shifted to higher-order regulatory features, i.e. chromatin accessibility, transcription factor (TF) binding and 3D chromatin interactions. This revealed at least two enhancer types: “lead” enhancers in which the presence of genetic variants modulates the activity of entire chromatin domains, and “dependent” ones in which variants induce subtle changes, affecting DNA accessibility, but not transcription. Although cell type-specific TFs are likely important, it remains unclear which sequence features are required to establish such enhancer hierarchies, and under which circumstances genetic variation results in altered enhancer-promoter contacts and differential gene expression. Here, we propose to investigate the molecular mechanisms that link DNA variation to TF binding, chromatin topology, and gene expression response. We will leverage data on enhancer hierarchy and sequence-specific TF binding to identify the sequence signatures that define “lead” enhancers. The results will guide the design of a synthetic locus that serves as an in vivo platform to systematically vary the building blocks of local transcriptional units: i) DNA sequence – including variations in TF binding site affinity and syntax, ii) molecular interactions between TFs, and iii) chromatin conformation. To validate our findings, we will perform optical reconstruction of chromatin architecture for a select number of DNA variants. By simultaneously perturbing co-dependent features, this proposal will provide novel mechanistic insights into the formation of local transcriptional hubs.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-RI","coordinator":"ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229288","id":"898218","acronym":"devUTRs","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Uncovering the roles of 5′UTRs in translational control during early zebrafish development","startDate":"2021-09-01","endDate":"2023-08-31","projectUrl":"","objective":"Following fertilisation, metazoan embryos are transcriptionally silent, and embryogenesis is controlled by maternally deposited factors. Developmental progression requires the synthesis of new mRNAs and proteins in a coordinated fashion. Many posttranscriptional mechanisms regulate the fate of maternal mRNAs, but it is less understood how translational control shapes early embryogenesis. In eukaryotes, translation starts at the mRNA 5′ end, consisting of the 5′ cap and 5′ untranslated region (UTR). Protein synthesis is primarily regulated at the translation initiation step by elements within the 5′UTR. However, the role of 5′UTRs in regulating the dynamics of mRNA translation during vertebrate embryogenesis remains unexplored. For example, all vertebrate ribosomal protein (RP) mRNAs harbor a conserved terminal oligopyrimidine tract (TOP) in their 5′UTR. RP levels must be tightly controlled to ensure proper organismal development, but if and how the TOP motif mediates RP mRNA translational regulation during embryogenesis is unclear. Overall, we lack a systematic understanding of the regulatory information contained in 5′UTRs. In this work, I aim to uncover the 5′UTR in vivo rules for mRNA translational regulation during zebrafish embryogenesis. I propose to apply imaging and biochemical approaches to characterise the role of the TOP motif in RP mRNA translational regulation during embryogenesis and identify the trans-acting factor(s) that bind(s) to it (Aim 1). To systematically assess the contribution of 5′UTRs to mRNA translational regulation during zebrafish embryogenesis, I will couple a massively parallel reporter assay of 5′UTRs to polysome profiling (Aim 2). By integrating the translational behaviour of 5′UTR reporters throughout embryogenesis with sequence-based regression models, I anticipate to uncover novel cis-regulatory elements in 5′UTRs with developmental roles.","totalCost":"191149,44","ecMaxContribution":"191149,44","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITAT BASEL","coordinatorCountry":"CH","participants":"","participantCountries":"","subjects":""} +{"rcn":"229261","id":"893787","acronym":"HOLYHOST","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Welfare and Hosting buildings in the “Holy Land” between the 4th and the 7th c. AD","startDate":"2020-10-01","endDate":"2022-09-30","projectUrl":"","objective":"Between the 4th and the 7th century AD, many hospices dedicated to the poor, elderly, strangers and travelers were built in the countryside, along roads, around and inside cities. They were commissioned by the Church, rich pious men and women concerned by the redeem of their sins, as well as emperors who saw this as a guarantee of social stability. Welfare is thus an important phenomena of Late Antiquity, abundantly mentioned by ancient literary sources and inscriptions, particularly in the eastern part of the Empire. However, the buildings that provided shelter and care to the needy have not yet received sufficient attention from archaeologists. Except for buildings which were identified by their inventors as hostels dedicated to pilgrims, they are still invisible in the field. \nThe aim of the HOLYHOST research project is to bring this social history’s main topic on the field of archaeology. It will address the welfare issue through the archaeological and architectural survey and study of Ancient welfare and hosting establishments’ remains, in the Holy Land (Palestine and Jordan) and around. This work will contribute to a better understanding of the practices linked to hospitality, welfare, accommodation and care in Antiquity. Moreover, such establishments served as models for medieval and modern Islamic, Jewish and Christian waqf institutions (religious endowment), and welfare continues to be highly relevant nowadays, through issues still at the heart of contemporary challenges debated in Europe: poverty, social exclusion, migrant crisis, principle of reception and hospitality. This interdisciplinary and diachronic research project will thus offer many new research perspectives, in terms of history of architecture, evolution of care practices, social and political regulations.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSITE PARIS I PANTHEON-SORBONNE","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229282","id":"896189","acronym":"MICADO","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Microbial contribution to continental wetland carbon budget","startDate":"2021-01-04","endDate":"2023-01-03","projectUrl":"","objective":"Continental wetlands are major carbon dioxide sinks but the second largest source of methane. Monitoring of wetland methane emissions revealed large inter-site variability that is hard to explain in the framework of current biogeochemical theories. Methane production in wetlands is an anaerobic microbial driven process involving a complex set of microbial metabolisms depending on the availability of (i) energy (via the presence of specific redox couples), (ii) organic substrates and (iii) specific microbial communities. To understand the complexity of microbial drivers on wetland methane emissions and quantify their contribution, the MICADO project will set up a multidisciplinary approach linking isotope organic geochemistry and environmental microbiology to assess microbial functioning in situ. As an organic geochemist I have developed an innovative approach to trace in situ microbial activity via compound specific carbon isotope analysis of microbe macromolecules and organic metabolites. The host institution is a leader in France in environmental microbiology and biogeochemistry developing high-throughput metagenomics and microbial rate assessments, for which I will be trained during the MICADO project. These techniques are highly complementary and combined they will provide a comprehensive knowledge on microbial metabolisms involved in organic matter degradation encompassing their complexity and interactions. This will revisit the relationships between organic substrate availability and microbial communities and will contribute at estimating the impact of microbial activity on wetland methane emissions. This project will give me the opportunity to acquire fundamental knowledge and to develop original lines of research that will consolidate my position as an independent scientist in biogeochemistry.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"CENTRE NATIONAL DE LA RECHERCHE SCIENTIFIQUE CNRS","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229249","id":"891624","acronym":"CuTAN","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"Copper-Catalyzed Multicomponent Reactions in Tandem Processes for Target Molecule Synthesis","startDate":"2021-02-01","endDate":"2023-01-31","projectUrl":"","objective":"The invention of processes that can form several bonds, stereocentres and rings in a single process is key to a sustainable future in synthetic chemistry. Multicomponent reactions and tandem procedures are two strategies that enable the rapid build-up of molecular complexity from simple reagents. By combining these two strategies into a single procedure, the diversity, complexity and value of products can be further enhanced along with the efficiency and economy of their construction. In this project, Dr Satpathi will develop novel copper-catalyzed multicomponent couplings of unsaturated hydrocarbons (e.g. allenes, enynes) with imines and boron reagents. These procedures will provide high-value amine products with universally high regio-, diastero- and enantiocontrol. The products will bear a variety of synthetic handles, for example, amino, alkynyl/alkenyl, and boryl groups, thus the products are primed for subsequent transformation. Dr Satpathi will exploit this functionality in tandem intramolecular couplings (e.g. intramolecular Suzuki/Buchwald-Hartwig reactions) to provide core cyclic structures of drug molecules and natural products. Thus, through a tandem procedure of; 1) copper-catalyzed borofunctionalization, and; 2) subsequent transition-metal catalyzed cyclization, he will gain efficient access to highly sought-after complex molecules. Overall, the process will provide high-value, chiral, cyclic motifs from abundant, achiral, linear substrates. Finally, Dr Satpathi has identified the phthalide-isoquinoline family of alkaloids as target molecules to display the power of his tandem methodology. Dr Satpathi has devised a novel route, which begins with our tandem multifunctionalization/cyclization reaction, to provide a range of these important alkaloids. The chosen alkaloids are of particular interest as they display a range of bioactivities – for example as natural products, receptor antagonists and on-market drugs.","totalCost":"212933,76","ecMaxContribution":"212933,76","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"THE UNIVERSITY OF MANCHESTER","coordinatorCountry":"UK","participants":"","participantCountries":"","subjects":""} +{"rcn":"229239","id":"887259","acronym":"ALEHOOP","status":"SIGNED","programme":"H2020-EU.2.1.4.;H2020-EU.3.2.6.","topics":"BBI-2019-SO3-D3","frameworkProgramme":"H2020","title":"Biorefineries for the valorisation of macroalgal residual biomass and legume processing by-products to obtain new protein value chains for high-value food and feed applications","startDate":"2020-06-01","endDate":"2024-05-31","projectUrl":"","objective":"ALEHOOP provides the demonstration at pilot scale of both sustainable macroalgae and legume-based biorefineries for the recovery of low-cost dietary proteins from alga-based and plant residual biomass and their validation to meet market requirements of consumers and industry in the food and feed sectors. In these sectors, consumers are demanding affordable functional natural proteins from alternative sources and industry is demanding low-cost bio-based protein formulations with better performance and higher sustainability. \nCurrent protein demand for the 7.3 billion inhabitants of the world is approximately 202 Mt. Due to the rise in meat consumption more proteins are therefore required for animal feeding. To satisfy the current protein demand, Europe imports over 30 Mt of soy from the Americas each year mainly for animal feeding, entailing 95% dependency of EU on imported soy. Current sources of proteins are becoming unsustainable from an economic and environmental perspective for Europe resulting in concerns for sustainability and food security and leading to search for new alternative proteins. \nALEHOOP addresses the obtaining of proteins from green macroalgal blooms, brown seaweed by-products from algae processors and legume processing by-products (peas, lupines, beans and lentils) as alternative protein sources for animal feeding (case of green seaweed) and food applications (case of brown seaweed and legume by-products), since they are low cost and under-exploited biomass that do not compete with traditional food crops for space and resources. This will reduce EU´s dependency on protein imports and contribute to our raw material security. The new proteins will be validated in foods for elderly, sporty and overweight people, vegetarians and healthy consumers as well as for animal feed creating cross-sectorial interconnection between these value chains and supporting the projected business plan.","totalCost":"6718370","ecMaxContribution":"5140274,41","call":"H2020-BBI-JTI-2019","fundingScheme":"BBI-IA-DEMO","coordinator":"CONTACTICA S.L.","coordinatorCountry":"ES","participants":"CENTIV GMBH;ALGINOR ASA;FUNDACION TECNALIA RESEARCH & INNOVATION;INDUKERN,S.A.;ASOCIACION NACIONAL DE FABRICANTES DE CONSERVAS DE PESCADOS Y MARISCOS-CENTRO TECNICO NACIONAL DE CONSERVACION DE PRODUCTOS DE LA PESCA;BIOZOON GMBH;EIGEN VERMOGEN VAN HET INSTITUUT VOOR LANDBOUW- EN VISSERIJONDERZOEK;BIOSURYA SL;VYZKUMNY USTAV VETERINARNIHO LEKARSTVI;NUTRITION SCIENCES;TECHNOLOGICAL UNIVERSITY DUBLIN;GARLAN, S.COOP.;ISANATUR SPAIN SL;UNIVERSIDAD DE VIGO;UNIVERSIDAD DE CADIZ","participantCountries":"DE;NO;ES;BE;CZ;IE","subjects":""} +{"rcn":"229258","id":"892834","acronym":"DENVPOC","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"qPCR Microfluidics point-of-care platform for dengue diagnosis","startDate":"2020-05-18","endDate":"2022-05-17","projectUrl":"","objective":"As a result of Global climate change and fast urbanization, global outbreaks of Dengue (DENV)/ Zika(ZIKV)/Chikungunya(CHIKV) virus have the potential to occur. The most common pathway of these infections in humans is through the female Aedes mosquito vector. DENV is an exanthematous febrile disease with varied clinical manifestations and progressions . Due to similarities in symptoms between DENV and ZIKV and CHIKV, it is difficult to make a differential diagnosis, impeding appropriate, timely medical intervention. Furthermore, cross-reactivity with ZIKV, which was recently related to microcephaly, is a serious issue. In 2016, in Brazil alone, there were 4180 microcephaly cases reported instead of 163 cases, more in line with yearly expected projections , , Thus, the sooner an accurate diagnostic which differentiates DENV from the other manifestations is critical; most especially at the early stages of the infection, to have a reliable diagnosis in pregnant women. In 2016, the OMS emergency committee declared that the outbreaks and the potentially resultant neurological disorders in Brazil were an important international state of emergency in public health, as a result of the associated secondary effects; these diseases became a Global concern. This project allows developing a highly and fast Multiplex qPCR POC platform by using FASTGENE technology with a minimal amount of patient serotype. It would reduce the time of analysis (30 to 90’ for a standard) and costs. Additionally, the sample preprocessing and thermalization will shorten real-time PCR amplification time and will be integrated within the microfluidic systems. This platform can result in a commercialized product whereupon a main market target would be pregnant women and people living or traveling through/from outbreak risk areas.","totalCost":"196707,84","ecMaxContribution":"196707,84","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-SE","coordinator":"BFORCURE","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229280","id":"895716","acronym":"DoMiCoP","status":"SIGNED","programme":"H2020-EU.1.3.2.","topics":"MSCA-IF-2019","frameworkProgramme":"H2020","title":"The Diffusion of Migration Control Practice. Actors, Processes and Effects.","startDate":"2021-03-01","endDate":"2023-02-28","projectUrl":"","objective":"DoMiCoP develops new understandings and perspectives to study migration control in practice in the European Union by asking one main question: how and why do communities of practice develop and diffuse the knowledge required to put migration control into action? Unlike the nexus between expert knowledge, epistemic communities and policy formulation, the nexus between everyday knowledge, communities of practice and policy implementation has not yet received systematic scholarly attention. My project bridges that gap by focusing on intermediate arenas in which communities of practice take shape most notably the meetings and trainings that gather state and non-state actors involved in putting asylum, detention and removal into practice. By building on field-based methodologies (interviews and participant observations), DoMiCoP sheds ethnographic light on the role that ‘learning from abroad’ plays in the implementation of migration control in the EU. My project’s aim is threefold: 1) Identifying arenas at intermediate levels in which communities of practice take shape; 2) Analysing the communities of practice by focusing on the configurations of actors and organizations involved, the motivations underlying their involvement, the process of knowledge development in interaction, the conflicts and negotiations; 3) Revealing the role of non-state organizations (private for profit and not-for-profit). From a theoretical point of view, this project goes beyond the classical view of the implementation as a test to assess the effectiveness of policy transfers towards an analysis of policy transfer at that level of policy-making. From an empirical point of view, the project expands knowledge about less-studied venues of policy-making and provides original thick descriptions. From a methodological point of view, the project engages with qualitative methods for the study of policy diffusion and aims at responding to their main challenges through participant observation.","totalCost":"163673,28","ecMaxContribution":"163673,28","call":"H2020-MSCA-IF-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"EUROPEAN UNIVERSITY INSTITUTE","coordinatorCountry":"IT","participants":"","participantCountries":"","subjects":""} +{"rcn":"229297","id":"954782","acronym":"MiniLLock","status":"SIGNED","programme":"H2020-EU.3.;H2020-EU.2.3.;H2020-EU.2.1.","topics":"EIC-SMEInst-2018-2020","frameworkProgramme":"H2020","title":"Mini Launch Lock devices for small satellites","startDate":"2020-05-01","endDate":"2022-04-30","projectUrl":"","objective":"Space industry is experiencing the most important paradigm shift in its history with the rise of small satellites and megaconstellations.\nSatellite miniaturization requires to reduce significantly production and orbit launching costs. To address the\nnew challenge of this manufacturing process and switch from craftsmanship to industrialization, space industry is turning\ntowards other domains looking for new solutions, disruptive technologies, and manufacturing process.\nMini Launch Lock devices for small satellites (MiniLLock) proposes innovative actuators on the cutting edge of customer\ndemand. They offer plug and play solutions that can directly be integrated into industry for satellites robotized production.\nMiniLLock is smaller, lighter, safer, with a longer lifetime and generates significantly less shocks and vibrations than\nstandard actuators such as electromagnet and pyrotechnics. MiniLLock offers performances which have never been reached\nwith any other materials.\nNimesis is the only company that can provide such cost-effective actuators suitable to small satellite with high performances\nand reliability, enabling features previously impossible.\nMiniLLock will accelerate and leverage the commercialization of Nimesis technology and ensure Europe worldwide\nleadership\nand independence in the new space emergent environment.\nNimesis ambitions to become the global leader of this domain with a turnover of € 26 million and a market share of 28% in\n2027.","totalCost":"2413543,75","ecMaxContribution":"1689480,63","call":"H2020-EIC-SMEInst-2018-2020-3","fundingScheme":"SME-2b","coordinator":"NIMESIS TECHNOLOGY SARL","coordinatorCountry":"FR","participants":"","participantCountries":"","subjects":""} +{"rcn":"229299","id":"101003374","acronym":"NOPHOS","status":"SIGNED","programme":"H2020-EU.4.","topics":"WF-02-2019","frameworkProgramme":"H2020","title":"Unravelling protein phosphorylation mechanisms and phosphoproteome changes under nitrosative stress conditions in E.coli","startDate":"2020-07-01","endDate":"2022-06-30","projectUrl":"","objective":"Currently, we face a global antibiotic resistance crisis aggravated by the slow development of more effective and anti-resistance promoting therapeutical solutions. Protein phosphorylation (PP) has recently emerged as one of the major post-translational modification in bacteria, involved in the regulation of multiple physiological processes. In this MSCA individual fellowship application we aim to bridge the current gap in the field for prokaryotes by unravelling the unknown regulatory role of PP on proteins involved in nitrosative stress (NS) detoxification in the model bacterium E.coli. We propose to examine for the first time both global protein modifications (e.g. phosphoproteomics) under nitrogen species stress, as well as characterize PP in individual proteins involved in NS response. We will construct a network model that reflect the phosphoproteomic changes upon NS in E.coli, that may pave the way for the design of new bacterial targets. Understanding how bacteria respond to the chemical weapons of the human innate system is fundamental to develop efficient therapies. We will pioneer research on the mechanism and the regulation of nitric oxide detoxification proteins already identified as phosphorylated, by analyzing how this modification influences their stability and activity in vitro and in vivo. This project opens up new research paths on bacterial detoxification systems and signalling in general, addressing for the first time the role of PP in these processes. The proposal brings together transversal and scientific skills that will enable the researcher to lead the development of this emerging field and position herself as an expert in the area, and aims at establishing the importance of PP in NO microbial response, a novelty in this field.","totalCost":"147815,04","ecMaxContribution":"147815,04","call":"H2020-WF-02-2019","fundingScheme":"MSCA-IF-EF-ST","coordinator":"UNIVERSIDADE NOVA DE LISBOA","coordinatorCountry":"PT","participants":"","participantCountries":"","subjects":""} \ No newline at end of file From 4589c428b142a833ab015990147310828d3c4832 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 21 May 2020 16:30:39 +0200 Subject: [PATCH 26/88] generate action sets and saves them in the hdfs path for the actions sets --- .../project/SparkAtomicActionJob.java | 71 ++++++++++++------- .../project/oozie_app/workflow.xml | 1 - 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java index 61bd952db..5b038b49a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java @@ -3,11 +3,16 @@ package eu.dnetlib.dhp.actionmanager.project; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; +import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.Optional; +import java.util.function.Consumer; +import eu.dnetlib.dhp.schema.action.AtomicAction; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; @@ -28,6 +33,15 @@ import eu.dnetlib.dhp.schema.oaf.Programme; import eu.dnetlib.dhp.schema.oaf.Project; import eu.dnetlib.dhp.utils.DHPUtils; +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.io.SequenceFile; +import org.apache.spark.rdd.SequenceFileRDDFunctions; +import org.apache.hadoop.io.Text; +import scala.Function1; +import scala.Tuple2; +import scala.runtime.BoxedUnit; + public class SparkAtomicActionJob { private static final Logger log = LoggerFactory.getLogger(SparkAtomicActionJob.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -61,6 +75,8 @@ public class SparkAtomicActionJob { final String programmePath = parser.get("programmePath"); log.info("programmePath {}: ", programmePath); + final String nameNode = parser.get("hdfsNameNode"); + SparkConf conf = new SparkConf(); runWithSparkSession( @@ -72,7 +88,8 @@ public class SparkAtomicActionJob { spark, projectPath, programmePath, - outputPath); + outputPath, + nameNode); }); } @@ -82,38 +99,38 @@ public class SparkAtomicActionJob { private static void getAtomicActions(SparkSession spark, String projectPatH, String programmePath, - String outputPath) { + String outputPath, + String nameNode) throws Exception{ Dataset project = readPath(spark, projectPatH, CSVProject.class); Dataset programme = readPath(spark, programmePath, CSVProgramme.class); project - .joinWith(programme, project.col("programme").equalTo(programme.col("code")), "left") - .map(c -> { - CSVProject csvProject = c._1(); - Optional csvProgramme = Optional.ofNullable(c._2()); - if (csvProgramme.isPresent()) { - Project p = new Project(); - p - .setId( - createOpenaireId( - ModelSupport.entityIdPrefix.get("project"), - "corda__h2020", csvProject.getId())); - Programme pm = new Programme(); - pm.setCode(csvProject.getProgramme()); - pm.setDescription(csvProgramme.get().getShortTitle()); - p.setProgramme(Arrays.asList(pm)); - return p; - } + .joinWith(programme, project.col("programme").equalTo(programme.col("code")), "left") + .map(c -> { + CSVProject csvProject = c._1(); + Optional csvProgramme = Optional.ofNullable(c._2()); + if (csvProgramme.isPresent()) { + Project p = new Project(); + p + .setId( + createOpenaireId( + ModelSupport.entityIdPrefix.get("project"), + "corda__h2020", csvProject.getId())); + Programme pm = new Programme(); + pm.setCode(csvProject.getProgramme()); + pm.setDescription(csvProgramme.get().getShortTitle()); + p.setProgramme(Arrays.asList(pm)); + return new AtomicAction<>(Project.class, p); + } + + return null; + }, Encoders.bean(AtomicAction.class)) + .filter(aa -> !(aa == null)) + .toJavaRDD() + .mapToPair(aa->new Tuple2<>(aa.getClazz().getCanonicalName(), OBJECT_MAPPER.writeValueAsString(aa))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, null); - return null; - }, Encoders.bean(Project.class)) - .filter(p -> !(p == null)) - // .map(p -> new AtomicAction<>(Project.class, p), Encoders.bean(AtomicAction.class)) - .write() - .option("compression", "gzip") - .mode(SaveMode.Overwrite) - .json(outputPath); } public static Dataset readPath( diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index cd4d79ab7..ba99fb314 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -24,7 +24,6 @@ - From 473c6d3a23c83053ffc7c13a81a40814f531b467 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 22 May 2020 15:26:57 +0200 Subject: [PATCH 27/88] produces AtomicActions instead of Projects --- .../project/SparkAtomicActionJob.java | 71 +++++++++---------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java index 5b038b49a..990e50abd 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java @@ -9,12 +9,17 @@ import java.util.HashMap; import java.util.Optional; import java.util.function.Consumer; -import eu.dnetlib.dhp.schema.action.AtomicAction; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; +import org.apache.hadoop.mapred.TextOutputFormat; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.rdd.SequenceFileRDDFunctions; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SaveMode; @@ -28,16 +33,11 @@ import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.Programme; import eu.dnetlib.dhp.schema.oaf.Project; import eu.dnetlib.dhp.utils.DHPUtils; - -import org.apache.hadoop.conf.Configuration; - -import org.apache.hadoop.io.SequenceFile; -import org.apache.spark.rdd.SequenceFileRDDFunctions; -import org.apache.hadoop.io.Text; import scala.Function1; import scala.Tuple2; import scala.runtime.BoxedUnit; @@ -75,8 +75,6 @@ public class SparkAtomicActionJob { final String programmePath = parser.get("programmePath"); log.info("programmePath {}: ", programmePath); - final String nameNode = parser.get("hdfsNameNode"); - SparkConf conf = new SparkConf(); runWithSparkSession( @@ -88,8 +86,7 @@ public class SparkAtomicActionJob { spark, projectPath, programmePath, - outputPath, - nameNode); + outputPath); }); } @@ -99,37 +96,39 @@ public class SparkAtomicActionJob { private static void getAtomicActions(SparkSession spark, String projectPatH, String programmePath, - String outputPath, - String nameNode) throws Exception{ + String outputPath) { Dataset project = readPath(spark, projectPatH, CSVProject.class); Dataset programme = readPath(spark, programmePath, CSVProgramme.class); project - .joinWith(programme, project.col("programme").equalTo(programme.col("code")), "left") - .map(c -> { - CSVProject csvProject = c._1(); - Optional csvProgramme = Optional.ofNullable(c._2()); - if (csvProgramme.isPresent()) { - Project p = new Project(); - p - .setId( - createOpenaireId( - ModelSupport.entityIdPrefix.get("project"), - "corda__h2020", csvProject.getId())); - Programme pm = new Programme(); - pm.setCode(csvProject.getProgramme()); - pm.setDescription(csvProgramme.get().getShortTitle()); - p.setProgramme(Arrays.asList(pm)); - return new AtomicAction<>(Project.class, p); - } + .joinWith(programme, project.col("programme").equalTo(programme.col("code")), "left") + .map(c -> { + CSVProject csvProject = c._1(); + Optional csvProgramme = Optional.ofNullable(c._2()); + if (csvProgramme.isPresent()) { + Project p = new Project(); + p + .setId( + createOpenaireId( + ModelSupport.entityIdPrefix.get("project"), + "corda__h2020", csvProject.getId())); + Programme pm = new Programme(); + pm.setCode(csvProject.getProgramme()); + pm.setDescription(csvProgramme.get().getShortTitle()); + p.setProgramme(Arrays.asList(pm)); + return p; + } - return null; - }, Encoders.bean(AtomicAction.class)) - .filter(aa -> !(aa == null)) - .toJavaRDD() - .mapToPair(aa->new Tuple2<>(aa.getClazz().getCanonicalName(), OBJECT_MAPPER.writeValueAsString(aa))) - .saveAsHadoopFile(outputPath, Text.class, Text.class, null); + return null; + }, Encoders.bean(Project.class)) + .filter(p -> !(p == null)) + .toJavaRDD() + .map(p -> new AtomicAction(Project.class, p)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); } From 50ad83b97f7190642f2e7a89e2cfbad53592b4f8 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 22 May 2020 15:27:19 +0200 Subject: [PATCH 28/88] - --- .../actionmanager/project/PrepareProjectTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java index fa9e32fd5..73bedb741 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java @@ -1,9 +1,10 @@ package eu.dnetlib.dhp.actionmanager.project; -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; -import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; @@ -18,9 +19,10 @@ import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme; +import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject; public class PrepareProjectTest { From ac8025f4696d7cf5b786ef9e0bc76620350748c5 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 22 May 2020 15:29:41 +0200 Subject: [PATCH 29/88] - --- .../dhp/actionmanager/project/SparkUpdateProjectTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index 3e559228f..f7b8722c4 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -70,7 +70,9 @@ public class SparkUpdateProjectTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-programmePath", - getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz").getPath(), + getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz") + .getPath(), "-projectPath", getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/prepared_projects.json").getPath(), "-outputPath", From f3dcca0dd05d3f2fd0796fe47dd98a163da62058 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 27 May 2020 17:23:34 +0200 Subject: [PATCH 30/88] added equals for programme --- .../eu/dnetlib/dhp/schema/oaf/Programme.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java index 69223ab01..c5259d07e 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java @@ -1,7 +1,10 @@ package eu.dnetlib.dhp.schema.oaf; -public class Programme { +import java.io.Serializable; +import java.util.Objects; + +public class Programme implements Serializable { private String code; private String description; @@ -20,4 +23,17 @@ public class Programme { public void setDescription(String description) { this.description = description; } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Programme programme = (Programme) o; + return Objects.equals(code, programme.code); + } + + } From dd1e0b93b851cde75ef27e53d743a44ea901ebb5 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 27 May 2020 17:40:32 +0200 Subject: [PATCH 31/88] added merge for Programme --- .../src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java index 1eae3e8ee..1fcfb305e 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java @@ -330,6 +330,9 @@ public class Project extends OafEntity implements Serializable { fundedamount = p.getFundedamount() != null && compareTrust(this, e) < 0 ? p.getFundedamount() : fundedamount; + + programme = mergeLists(programme, p.getProgramme()); + mergeOAFDataInfo(e); } } From 18554534340ce5ca7d53addeaed5c48c8d8ab883 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 27 May 2020 17:59:36 +0200 Subject: [PATCH 32/88] changed the outputdir of the last step --- .../eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index ba99fb314..421df460c 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -120,7 +120,7 @@ --projectPath${workingDir}/preparedProjects --programmePath${workingDir}/preparedProgramme - --outputPath/tmp/h2020programme + --outputPath${outputPath} From 669c05c7712154e37fa5d76fc8ea1a0bec9c9d8c Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 10:00:45 +0200 Subject: [PATCH 33/88] added groupBy before creating Actions --- .../project/SparkAtomicActionJob.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java index 990e50abd..1023e2d19 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java @@ -6,6 +6,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; +import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; @@ -19,6 +20,7 @@ import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextOutputFormat; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.api.java.function.MapGroupsFunction; import org.apache.spark.rdd.SequenceFileRDDFunctions; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; @@ -122,7 +124,17 @@ public class SparkAtomicActionJob { return null; }, Encoders.bean(Project.class)) - .filter(p -> !(p == null)) + .filter(Objects::nonNull) + .groupByKey( + (MapFunction) p -> p.getId(), + Encoders.STRING()) + .mapGroups((MapGroupsFunction) (s, it) -> { + Project first = it.next(); + it.forEachRemaining(p -> { + first.mergeFrom(p); + }); + return first; + }, Encoders.bean(Project.class)) .toJavaRDD() .map(p -> new AtomicAction(Project.class, p)) .mapToPair( From 96d1a3c4316f972924a7c0a76dc0d7f066a1712c Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 10:04:10 +0200 Subject: [PATCH 34/88] deleted the file were to store the csv files --- .../dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java index 905194232..2b72b229a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/csvutils/ReadCSV.java @@ -76,10 +76,9 @@ public class ReadCSV implements Closeable { Path hdfsWritePath = new Path(hdfsPath); FSDataOutputStream fsDataOutputStream = null; if (fileSystem.exists(hdfsWritePath)) { - fsDataOutputStream = fileSystem.append(hdfsWritePath); - } else { - fsDataOutputStream = fileSystem.create(hdfsWritePath); + fileSystem.delete(hdfsWritePath, false); } + fsDataOutputStream = fileSystem.create(hdfsWritePath); this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); this.csvFile = httpConnector.getInputSource(fileURL); From 1060977272fd90449c48612a7976511d9d525154 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 10:04:36 +0200 Subject: [PATCH 35/88] added fs actions to remove and the create the workingDir --- .../eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index 421df460c..edeb0582a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -24,6 +24,8 @@ + + From 87b07f4af88b5fa87d807dc5c15fad097409f7e5 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 10:05:43 +0200 Subject: [PATCH 36/88] removed unused variables --- .../eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java index a5abb9ea7..c6dab13a0 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java @@ -25,7 +25,6 @@ public class PrepareProgramme { private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final HashMap programmeMap = new HashMap<>(); public static void main(String[] args) throws Exception { From df44db686a8b53e2769167e6329d0f4b9d605cbf Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 10:07:00 +0200 Subject: [PATCH 37/88] refactoring --- .../project/PrepareProjects.java | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java index df37f9286..7ca50b219 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java @@ -3,10 +3,7 @@ package eu.dnetlib.dhp.actionmanager.project; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; +import java.util.*; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -81,17 +78,14 @@ public class PrepareProjects { .flatMap(p -> { List csvProjectList = new ArrayList<>(); String[] programme = p.getProgramme().split(";"); - if (programme.length > 1) { - String id = p.getId(); - for (int i = 0; i < programme.length; i++) { + Arrays + .stream(programme) + .forEach(value -> { CSVProject csvProject = new CSVProject(); - csvProject.setProgramme(programme[i]); - csvProject.setId(id); + csvProject.setProgramme(value); + csvProject.setId(p.getId()); csvProjectList.add(csvProject); - } - } else { - csvProjectList.add(p); - } + }); return csvProjectList.iterator(); }) From 35b72791478e42f3e1e1b1f708ddf825acb4dcc5 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 10:26:12 +0200 Subject: [PATCH 38/88] changed test because data are saved as SequenceFile now, and because of the group by the umber of produced update decrease --- .../actionmanager/project/SparkUpdateProjectTest.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index f7b8722c4..64c6ac32f 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -5,7 +5,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import eu.dnetlib.dhp.schema.action.AtomicAction; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.io.Text; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -82,10 +84,12 @@ public class SparkUpdateProjectTest { final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .textFile(workingDir.toString() + "/actionSet") - .map(item -> OBJECT_MAPPER.readValue(item, Project.class)); + .sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class) + .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) + .map(aa -> ((Project)aa.getPayload())) + ; - Assertions.assertEquals(16, tmp.count()); + Assertions.assertEquals(14, tmp.count()); // Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProgramme.class)); // From b737ed8236a8b1866a634e4edb25452e28dd9fe7 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 17:29:21 +0200 Subject: [PATCH 39/88] added part to read projects from the openaire db to filter out those in the csv file that are not in the db --- .../actionmanager/project/ProjectSubset.java | 16 +++ .../project/ReadProjectsFromDB.java | 113 ++++++++++++++++++ .../project/read_projects_db.json | 32 +++++ 3 files changed, 161 insertions(+) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java new file mode 100644 index 000000000..cfbb62f21 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java @@ -0,0 +1,16 @@ +package eu.dnetlib.dhp.actionmanager.project; + +import java.io.Serializable; + +public class ProjectSubset implements Serializable { + + private String code; + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java new file mode 100644 index 000000000..0015dc60f --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java @@ -0,0 +1,113 @@ +package eu.dnetlib.dhp.actionmanager.project; + +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.DbClient; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.sql.ResultSet; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; + +public class ReadProjectsFromDB implements Closeable { + + private final DbClient dbClient; + private static final Log log = LogFactory.getLog(ReadProjectsFromDB.class); + private final Configuration conf; + private final BufferedWriter writer; + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final static String query = "SELECT code " + + "from projects where id like 'corda__h2020%' " ; + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + ReadProjectsFromDB.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json"))); + + parser.parseArgument(args); + + final String dbUrl = parser.get("postgresUrl"); + final String dbUser = parser.get("postgresUser"); + final String dbPassword = parser.get("postgresPassword"); + final String hdfsPath = parser.get("hdfsPath") ; + final String hdfsNameNode = parser.get("hdfsNameNode"); + + try (final ReadProjectsFromDB rbl = new ReadProjectsFromDB(hdfsPath, hdfsNameNode, dbUrl, dbUser, + dbPassword)) { + + log.info("Processing blacklist..."); + rbl.execute(query, rbl::processProjectsEntry); + + } + } + public void execute(final String sql, final Function> producer) throws Exception { + + final Consumer consumer = rs -> producer.apply(rs).forEach(r -> writeProject(r)); + + dbClient.processResults(sql, consumer); + } + + public List processProjectsEntry(ResultSet rs) { + try { + ProjectSubset p = new ProjectSubset(); + p.setCode(rs.getString("code")); + + return Arrays.asList(p); + + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + protected void writeProject(final ProjectSubset r) { + try { + writer.write(OBJECT_MAPPER.writeValueAsString(r)); + writer.newLine(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + public ReadProjectsFromDB( + final String hdfsPath, String hdfsNameNode, final String dbUrl, final String dbUser, final String dbPassword) + throws Exception { + + this.dbClient = new DbClient(dbUrl, dbUser, dbPassword); + this.conf = new Configuration(); + this.conf.set("fs.defaultFS", hdfsNameNode); + FileSystem fileSystem = FileSystem.get(this.conf); + Path hdfsWritePath = new Path(hdfsPath); + FSDataOutputStream fsDataOutputStream = null; + if (fileSystem.exists(hdfsWritePath)) { + fileSystem.delete(hdfsWritePath, false); + } + fsDataOutputStream = fileSystem.create(hdfsWritePath); + + + this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); + } + + @Override + public void close() throws IOException { + dbClient.close(); + writer.close(); + } +} + diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json new file mode 100644 index 000000000..9a2eadaa7 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json @@ -0,0 +1,32 @@ +[ + { + "paramName": "p", + "paramLongName": "hdfsPath", + "paramDescription": "the path where storing the sequential file", + "paramRequired": true + }, + { + "paramName": "nn", + "paramLongName": "hdfsNameNode", + "paramDescription": "the name node on hdfs", + "paramRequired": true + }, + { + "paramName": "pgurl", + "paramLongName": "postgresUrl", + "paramDescription": "postgres url, example: jdbc:postgresql://localhost:5432/testdb", + "paramRequired": true + }, + { + "paramName": "pguser", + "paramLongName": "postgresUser", + "paramDescription": "postgres user", + "paramRequired": false + }, + { + "paramName": "pgpasswd", + "paramLongName": "postgresPassword", + "paramDescription": "postgres password", + "paramRequired": false + } +] \ No newline at end of file From 5309a99a70bf6e81ca5d55e6d0062be1d75e05f4 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 17:29:53 +0200 Subject: [PATCH 40/88] modified the PrepareProjects to consider those in the db --- .../project/PrepareProjects.java | 71 ++++++++++++++----- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java index 7ca50b219..3d8226f4d 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java @@ -9,9 +9,11 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +57,9 @@ public class PrepareProjects { final String outputPath = parser.get("outputPath"); log.info("outputPath {}: ", outputPath); + final String dbProjectPath = parser.get("dbProjectPath"); + log.info("dbProjectPath {}: ", dbProjectPath); + SparkConf conf = new SparkConf(); runWithSparkSession( @@ -62,7 +67,7 @@ public class PrepareProjects { isSparkSessionManaged, spark -> { removeOutputDir(spark, outputPath); - exec(spark, projectPath, outputPath); + exec(spark, projectPath, dbProjectPath, outputPath); }); } @@ -70,27 +75,55 @@ public class PrepareProjects { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } - private static void exec(SparkSession spark, String progjectPath, String outputPath) { + private static void exec(SparkSession spark, String progjectPath, String dbProjectPath, String outputPath) { Dataset project = readPath(spark, progjectPath, CSVProject.class); + Dataset dbProjects = readPath(spark, dbProjectPath, ProjectSubset.class); - project - .toJavaRDD() - .flatMap(p -> { - List csvProjectList = new ArrayList<>(); - String[] programme = p.getProgramme().split(";"); - Arrays - .stream(programme) - .forEach(value -> { - CSVProject csvProject = new CSVProject(); - csvProject.setProgramme(value); - csvProject.setId(p.getId()); - csvProjectList.add(csvProject); - }); + dbProjects.joinWith(project, dbProjects.col("code").equalTo(project.col("id")), "left") + .flatMap((FlatMapFunction, CSVProject>) value -> { + Optional csvProject = Optional.ofNullable(value._2()); + if(! csvProject.isPresent()){ + return null; + } + List csvProjectList = new ArrayList<>(); + String[] programme = csvProject.get().getProgramme().split(";"); + Arrays + .stream(programme) + .forEach(p -> { + CSVProject proj = new CSVProject(); + proj.setProgramme(p); + proj.setId(csvProject.get().getId()); + csvProjectList.add(proj); + }); - return csvProjectList.iterator(); - }) - .map(p -> OBJECT_MAPPER.writeValueAsString(p)) - .saveAsTextFile(outputPath); + return csvProjectList.iterator(); + }, Encoders.bean(CSVProject.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); +// +// .map(value -> { +// Optional csvProject = Optional.ofNullable(value._2()); +// }, Encoders.bean(CSVProject.class)) +// .filter(Objects::nonNull) +// .toJavaRDD() +// .flatMap(p -> { +// List csvProjectList = new ArrayList<>(); +// String[] programme = p.getProgramme().split(";"); +// Arrays +// .stream(programme) +// .forEach(value -> { +// CSVProject csvProject = new CSVProject(); +// csvProject.setProgramme(value); +// csvProject.setId(p.getId()); +// csvProjectList.add(csvProject); +// }); +// +// return csvProjectList.iterator(); +// }) +// .map(p -> OBJECT_MAPPER.writeValueAsString(p)) +// .saveAsTextFile(outputPath); } From 6a15067a644133d2178e50fa2578db49f03a0934 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 17:30:09 +0200 Subject: [PATCH 41/88] added one step in the workflow --- .../actionmanager/project/oozie_app/workflow.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index edeb0582a..ca0a73b97 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -51,6 +51,19 @@ --hdfsPath${workingDir}/programme --classForNameeu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme + + + + + + + eu.dnetlib.dhp.actionmanager.project.ReadProjectsFromDB + --hdfsPath${workingDir}/dbProjects + --hdfsNameNode${nameNode} + --postgresUrl${postgresURL} + --postgresUser${postgresUser} + --postgresPassword${postgresPassword} + From 773735f87059e43b1b700128b5777ca922149c45 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 17:30:45 +0200 Subject: [PATCH 42/88] added the path to the file containing the projects code from the db --- .../actionmanager/project/prepare_project_parameters.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json index 5fc88ce8e..49f9c7306 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/prepare_project_parameters.json @@ -16,5 +16,11 @@ "paramLongName": "outputPath", "paramDescription": "the path of the new ActionSet", "paramRequired": true -} +}, + { + "paramName": "dbp", + "paramLongName": "dbProjectPath", + "paramDescription": "the path of the project code read from db", + "paramRequired": true + } ] \ No newline at end of file From 01f7876595c7170ad5d7c1c7d9d6e13087b1f288 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 23:50:32 +0200 Subject: [PATCH 43/88] fix issue with flatMap - the return type must not be null --- .../project/PrepareProjects.java | 77 ++++++++----------- .../dhp/actionmanager/project/dbProject | 0 2 files changed, 30 insertions(+), 47 deletions(-) create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java index 3d8226f4d..78aed1a69 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java @@ -57,7 +57,7 @@ public class PrepareProjects { final String outputPath = parser.get("outputPath"); log.info("outputPath {}: ", outputPath); - final String dbProjectPath = parser.get("dbProjectPath"); + final String dbProjectPath = parser.get("dbProjectPath"); log.info("dbProjectPath {}: ", dbProjectPath); SparkConf conf = new SparkConf(); @@ -75,56 +75,39 @@ public class PrepareProjects { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } - private static void exec(SparkSession spark, String progjectPath, String dbProjectPath, String outputPath) { - Dataset project = readPath(spark, progjectPath, CSVProject.class); + private static void exec(SparkSession spark, String projectPath, String dbProjectPath, String outputPath) { + Dataset project = readPath(spark, projectPath, CSVProject.class); Dataset dbProjects = readPath(spark, dbProjectPath, ProjectSubset.class); - dbProjects.joinWith(project, dbProjects.col("code").equalTo(project.col("id")), "left") - .flatMap((FlatMapFunction, CSVProject>) value -> { - Optional csvProject = Optional.ofNullable(value._2()); - if(! csvProject.isPresent()){ - return null; - } - List csvProjectList = new ArrayList<>(); - String[] programme = csvProject.get().getProgramme().split(";"); - Arrays - .stream(programme) - .forEach(p -> { - CSVProject proj = new CSVProject(); - proj.setProgramme(p); - proj.setId(csvProject.get().getId()); - csvProjectList.add(proj); - }); + dbProjects + .joinWith(project, dbProjects.col("code").equalTo(project.col("id")), "left") + .flatMap(getTuple2CSVProjectFlatMapFunction(), Encoders.bean(CSVProject.class)) + .filter(Objects::nonNull) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); - return csvProjectList.iterator(); - }, Encoders.bean(CSVProject.class)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath); -// -// .map(value -> { -// Optional csvProject = Optional.ofNullable(value._2()); -// }, Encoders.bean(CSVProject.class)) -// .filter(Objects::nonNull) -// .toJavaRDD() -// .flatMap(p -> { -// List csvProjectList = new ArrayList<>(); -// String[] programme = p.getProgramme().split(";"); -// Arrays -// .stream(programme) -// .forEach(value -> { -// CSVProject csvProject = new CSVProject(); -// csvProject.setProgramme(value); -// csvProject.setId(p.getId()); -// csvProjectList.add(csvProject); -// }); -// -// return csvProjectList.iterator(); -// }) -// .map(p -> OBJECT_MAPPER.writeValueAsString(p)) -// .saveAsTextFile(outputPath); + } + private static FlatMapFunction, CSVProject> getTuple2CSVProjectFlatMapFunction() { + return (FlatMapFunction, CSVProject>) value -> { + Optional csvProject = Optional.ofNullable(value._2()); + List csvProjectList = new ArrayList<>(); + if (csvProject.isPresent()) { + + String[] programme = csvProject.get().getProgramme().split(";"); + Arrays + .stream(programme) + .forEach(p -> { + CSVProject proj = new CSVProject(); + proj.setProgramme(p); + proj.setId(csvProject.get().getId()); + csvProjectList.add(proj); + }); + } + return csvProjectList.iterator(); + }; } public static Dataset readPath( diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject new file mode 100644 index 000000000..e69de29bb From 782984d8e5a03cfc2093a76d9c45d458836db1a6 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 23:52:41 +0200 Subject: [PATCH 44/88] added needed parameter --- .../dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index ca0a73b97..1e3445675 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -96,7 +96,7 @@ yarn cluster - PrepareProgramme + PrepareProjects eu.dnetlib.dhp.actionmanager.project.PrepareProjects dhp-aggregation-${projectVersion}.jar @@ -111,6 +111,7 @@ --projectPath${workingDir}/projects --outputPath${workingDir}/preparedProjects + --dbProjectPath${workingDir}/dbProjects From 6989fb9c8ad6c5ee45979e7da82fe1d491927d5e Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 23:53:24 +0200 Subject: [PATCH 45/88] changed the project test according to the newly introduced join with the db project codes --- .../dhp/actionmanager/project/PrepareProjectTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java index 73bedb741..5ff88e46f 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjectTest.java @@ -67,7 +67,7 @@ public class PrepareProjectTest { } @Test - public void numberDistinctProgrammeTest() throws Exception { + public void numberDistinctProjectTest() throws Exception { PrepareProjects .main( new String[] { @@ -76,7 +76,10 @@ public class PrepareProjectTest { "-projectPath", getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/projects_subset.json").getPath(), "-outputPath", - workingDir.toString() + "/preparedProjects" + workingDir.toString() + "/preparedProjects", + "-dbProjectPath", + getClass().getResource("/eu/dnetlib/dhp/actionmanager/project/dbProject").getPath(), + }); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); @@ -85,7 +88,7 @@ public class PrepareProjectTest { .textFile(workingDir.toString() + "/preparedProjects") .map(item -> OBJECT_MAPPER.readValue(item, CSVProject.class)); - Assertions.assertEquals(20, tmp.count()); + Assertions.assertEquals(8, tmp.count()); Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProject.class)); From 8b6e886fb6d89681d00fc215c271c084f60ccbb9 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 28 May 2020 23:54:31 +0200 Subject: [PATCH 46/88] added new resource for testing --- .../eu/dnetlib/dhp/actionmanager/project/dbProject | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject index e69de29bb..f8e3c4589 100644 --- a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/dbProject @@ -0,0 +1,8 @@ +{"code":"894593"} +{"code":"897004"} +{"code":"896300"} +{"code":"892890"} +{"code":"886828"} +{"code":"8867767"} +{"code":"101003374"} +{"code":"886776"} \ No newline at end of file From 6f1eea28b69a10aeb4338305ae43a4b936494961 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 29 May 2020 10:41:39 +0200 Subject: [PATCH 47/88] changed message in log --- .../project/ReadProjectsFromDB.java | 160 +++++++++--------- 1 file changed, 81 insertions(+), 79 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java index 0015dc60f..2d541d2f9 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java @@ -1,15 +1,5 @@ -package eu.dnetlib.dhp.actionmanager.project; -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.DbClient; -import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; +package eu.dnetlib.dhp.actionmanager.project; import java.io.BufferedWriter; import java.io.Closeable; @@ -22,92 +12,104 @@ import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.DbClient; + public class ReadProjectsFromDB implements Closeable { - private final DbClient dbClient; - private static final Log log = LogFactory.getLog(ReadProjectsFromDB.class); - private final Configuration conf; - private final BufferedWriter writer; - private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final DbClient dbClient; + private static final Log log = LogFactory.getLog(ReadProjectsFromDB.class); + private final Configuration conf; + private final BufferedWriter writer; + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final static String query = "SELECT code " + - "from projects where id like 'corda__h2020%' " ; + private final static String query = "SELECT code " + + "from projects where id like 'corda__h2020%' "; - public static void main(final String[] args) throws Exception { - final ArgumentApplicationParser parser = new ArgumentApplicationParser( - IOUtils - .toString( - ReadProjectsFromDB.class - .getResourceAsStream( - "/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json"))); + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + ReadProjectsFromDB.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/read_projects_db.json"))); - parser.parseArgument(args); + parser.parseArgument(args); - final String dbUrl = parser.get("postgresUrl"); - final String dbUser = parser.get("postgresUser"); - final String dbPassword = parser.get("postgresPassword"); - final String hdfsPath = parser.get("hdfsPath") ; - final String hdfsNameNode = parser.get("hdfsNameNode"); + final String dbUrl = parser.get("postgresUrl"); + final String dbUser = parser.get("postgresUser"); + final String dbPassword = parser.get("postgresPassword"); + final String hdfsPath = parser.get("hdfsPath"); + final String hdfsNameNode = parser.get("hdfsNameNode"); - try (final ReadProjectsFromDB rbl = new ReadProjectsFromDB(hdfsPath, hdfsNameNode, dbUrl, dbUser, - dbPassword)) { + try (final ReadProjectsFromDB rbl = new ReadProjectsFromDB(hdfsPath, hdfsNameNode, dbUrl, dbUser, + dbPassword)) { - log.info("Processing blacklist..."); - rbl.execute(query, rbl::processProjectsEntry); + log.info("Processing projects..."); + rbl.execute(query, rbl::processProjectsEntry); - } - } - public void execute(final String sql, final Function> producer) throws Exception { + } + } - final Consumer consumer = rs -> producer.apply(rs).forEach(r -> writeProject(r)); + public void execute(final String sql, final Function> producer) throws Exception { - dbClient.processResults(sql, consumer); - } + final Consumer consumer = rs -> producer.apply(rs).forEach(r -> writeProject(r)); - public List processProjectsEntry(ResultSet rs) { - try { - ProjectSubset p = new ProjectSubset(); - p.setCode(rs.getString("code")); + dbClient.processResults(sql, consumer); + } - return Arrays.asList(p); + public List processProjectsEntry(ResultSet rs) { + try { + ProjectSubset p = new ProjectSubset(); + p.setCode(rs.getString("code")); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } + return Arrays.asList(p); - protected void writeProject(final ProjectSubset r) { - try { - writer.write(OBJECT_MAPPER.writeValueAsString(r)); - writer.newLine(); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } + } catch (final Exception e) { + throw new RuntimeException(e); + } + } - public ReadProjectsFromDB( - final String hdfsPath, String hdfsNameNode, final String dbUrl, final String dbUser, final String dbPassword) - throws Exception { + protected void writeProject(final ProjectSubset r) { + try { + writer.write(OBJECT_MAPPER.writeValueAsString(r)); + writer.newLine(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } - this.dbClient = new DbClient(dbUrl, dbUser, dbPassword); - this.conf = new Configuration(); - this.conf.set("fs.defaultFS", hdfsNameNode); - FileSystem fileSystem = FileSystem.get(this.conf); - Path hdfsWritePath = new Path(hdfsPath); - FSDataOutputStream fsDataOutputStream = null; - if (fileSystem.exists(hdfsWritePath)) { - fileSystem.delete(hdfsWritePath, false); - } - fsDataOutputStream = fileSystem.create(hdfsWritePath); + public ReadProjectsFromDB( + final String hdfsPath, String hdfsNameNode, final String dbUrl, final String dbUser, final String dbPassword) + throws Exception { + this.dbClient = new DbClient(dbUrl, dbUser, dbPassword); + this.conf = new Configuration(); + this.conf.set("fs.defaultFS", hdfsNameNode); + FileSystem fileSystem = FileSystem.get(this.conf); + Path hdfsWritePath = new Path(hdfsPath); + FSDataOutputStream fsDataOutputStream = null; + if (fileSystem.exists(hdfsWritePath)) { + fileSystem.delete(hdfsWritePath, false); + } + fsDataOutputStream = fileSystem.create(hdfsWritePath); - this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); - } + this.writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); + } - @Override - public void close() throws IOException { - dbClient.close(); - writer.close(); - } + @Override + public void close() throws IOException { + dbClient.close(); + writer.close(); + } } - From dfa4997a4ff96605cc91fc467e2b203ba1e3f3fc Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 29 May 2020 10:45:18 +0200 Subject: [PATCH 48/88] removed commented code --- .../actionmanager/project/SparkUpdateProjectTest.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index 64c6ac32f..4d3ec140b 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import eu.dnetlib.dhp.schema.action.AtomicAction; import org.apache.commons.io.FileUtils; import org.apache.hadoop.io.Text; import org.apache.spark.SparkConf; @@ -21,6 +20,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.oaf.Project; public class SparkUpdateProjectTest { @@ -84,15 +84,12 @@ public class SparkUpdateProjectTest { final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class) + .sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class) .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) - .map(aa -> ((Project)aa.getPayload())) - ; + .map(aa -> ((Project) aa.getPayload())); Assertions.assertEquals(14, tmp.count()); -// Dataset verificationDataset = spark.createDataset(tmp.rdd(), Encoders.bean(CSVProgramme.class)); -// -// Assertions.assertEquals(0, verificationDataset.filter("shortTitle =''").count()); + } } From b2349659cfceca0f749a75a388a012a3939ae1a0 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 5 Jun 2020 18:37:38 +0200 Subject: [PATCH 49/88] WIP: graph property fixing implementation --- .../dhp/oa/graph/fix/FixGraphProperties.java | 210 +++++++++++++ .../dhp/oa/graph/fix/config-default.xml | 18 ++ .../eu/dnetlib/dhp/oa/graph/fix/workflow.xml | 289 ++++++++++++++++++ .../oa/graph/input_fix_graph_parameters.json | 32 ++ 4 files changed, 549 insertions(+) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/fix/FixGraphProperties.java create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/config-default.xml create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/workflow.xml create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_fix_graph_parameters.json diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/fix/FixGraphProperties.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/fix/FixGraphProperties.java new file mode 100644 index 000000000..bbcadde59 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/fix/FixGraphProperties.java @@ -0,0 +1,210 @@ +package eu.dnetlib.dhp.oa.graph.fix; + +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.common.EntityType; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.*; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +public class FixGraphProperties { + + private static final Logger log = LoggerFactory.getLogger(FixGraphProperties.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + FixGraphProperties.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/input_fix_graph_parameters.json")); + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String inputPath = parser.get("inputPath"); + log.info("inputPath: {}", inputPath); + + String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); + + String graphTableClassName = parser.get("graphTableClassName"); + log.info("graphTableClassName: {}", graphTableClassName); + + Class entityClazz = (Class) Class.forName(graphTableClassName); + + final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookupUrl); + + SparkConf conf = new SparkConf(); + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + fixGraphTable(spark, vocs, inputPath, entityClazz, outputPath); + }); + } + + private static void fixGraphTable( + SparkSession spark, + VocabularyGroup vocs, + String inputPath, + Class clazz, + String outputPath) { + + MapFunction fixFn = getFixingFunction(vocs, clazz); + + readTableFromPath(spark, inputPath, clazz) + .map(fixFn, Encoders.bean(clazz)) + .write() + .mode(SaveMode.Overwrite) + .parquet(outputPath); + } + + private static MapFunction getFixingFunction(VocabularyGroup vocs, Class clazz) { + + switch (clazz.getCanonicalName()) { + case "eu.dnetlib.dhp.schema.oaf.Publication": + case "eu.dnetlib.dhp.schema.oaf.Dataset": + case "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct": + case "eu.dnetlib.dhp.schema.oaf.Software": + return (MapFunction) value -> { + Result r = (Result) value; + + if (r.getLanguage() != null) { + r.setLanguage(vocs.getTermAsQualifier("dnet:languages", "und")); + } else { + r.setLanguage(vocs.getTermAsQualifier("dnet:languages", r.getLanguage().getClassid())); + } + if (r.getCountry() != null) { + r.setCountry( + r.getCountry() + .stream() + .filter(Objects::nonNull) + .map(c -> { + Qualifier q = vocs.getTermAsQualifier("dnet:countries", c.getClassid()); + Country cn = new Country(); + cn.setDataInfo(c.getDataInfo()); + cn.setClassid(q.getClassid()); + cn.setClassname(cn.getClassname()); + cn.setSchemeid("dnet:countries"); + cn.setSchemename("dnet:countries"); + return cn; + }) + .collect(Collectors.toList())); + } + + if (r.getSubject() != null) { + r.setSubject( + r.getSubject() + .stream() + .filter(Objects::nonNull) + .map(s -> { + if (s.getQualifier() == null || StringUtils.isBlank(s.getQualifier().getClassid())) { + s.setQualifier(vocs.getTermAsQualifier("dnet:subject_classification_typologies", "UNKNOWN")); + } + }) + .collect(Collectors.toList()) + ); + } + + if (r.getPublisher() != null && StringUtils.isBlank(r.getPublisher().getValue())) { + r.setPublisher(null); + } + if (r.getBestaccessright() == null) { + r.setBestaccessright(vocs.getTermAsQualifier("dnet:access_modes", "UNKNOWN")); + } + if (r.getInstance() != null) { + for(Instance i : r.getInstance()) { + if (i.getAccessright() == null) { + i.setAccessright(vocs.getTermAsQualifier("dnet:access_modes", "UNKNOWN")); + } + if (i.getInstancetype() != null) { + i.setInstancetype(vocs.getTermAsQualifier("dnet:publication_resource", i.getInstancetype().getClassid())); + } else { + i.setInstancetype(vocs.getTermAsQualifier("dnet:publication_resource", "0000")); + } + + + } + } + + return clazz.cast(r); + }; + case "eu.dnetlib.dhp.schema.oaf.Datasource": + return (MapFunction) value -> { + return value; + }; + case "eu.dnetlib.dhp.schema.oaf.Organization": + return (MapFunction) value -> { + Organization o = (Organization) value; + + if (o.getCountry() == null) { + o.setCountry(vocs.getTermAsQualifier("dnet:countries", "UNKNOWN")); + } else { + o.setCountry(vocs.getTermAsQualifier("dnet:countries", o.getCountry().getClassid())); + } + + return clazz.cast(o); + }; + case "eu.dnetlib.dhp.schema.oaf.Project": + return (MapFunction) value -> { + return value; + }; + case "eu.dnetlib.dhp.schema.oaf.Relation": + return (MapFunction) value -> { + return value; + }; + default: + throw new RuntimeException("unknown class: " + clazz.getCanonicalName()); + } + + } + + private static Dataset readTableFromPath( + SparkSession spark, String inputEntityPath, Class clazz) { + + log.info("Reading Graph table from: {}", inputEntityPath); + return spark + .read() + .textFile(inputEntityPath) + .map( + (MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), + Encoders.bean(clazz)); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/config-default.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/config-default.xml new file mode 100644 index 000000000..2e0ed9aee --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/config-default.xml @@ -0,0 +1,18 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/workflow.xml new file mode 100644 index 000000000..e93f58279 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/workflow.xml @@ -0,0 +1,289 @@ + + + + + graphInputPath + the input path to read graph content + + + graphOutputPath + the target path to store fixed graph + + + isLookupUrl + the address of the lookUp service + + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + oozieActionShareLibForSpark2 + oozie action sharelib for spark 2.* + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + spark 2.* extra listeners classname + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + spark 2.* sql query execution listeners classname + + + spark2YarnHistoryServerAddress + spark 2.* yarn history server address + + + spark2EventLogDir + spark 2.* event log dir location + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + + yarn + cluster + Fix publications + eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputPath${graphInputPath}/publication + --outputPath${graphOutputPath}/publication + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Fix datasets + eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputPath${graphInputPath}/dataset + --outputPath${graphOutputPath}/dataset + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Dataset + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Fix otherresearchproducts + eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputPath${graphInputPath}/otherresearchproduct + --outputPath${graphOutputPath}/otherresearchproduct + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Fix softwares + eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputPath${graphInputPath}/software + --outputPath${graphOutputPath}/software + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Software + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Fix datasources + eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputPath${graphInputPath}/datasource + --outputPath${graphOutputPath}/datasource + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Datasource + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Fix organizations + eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputPath${graphInputPath}/organization + --outputPath${graphOutputPath}/organization + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Organization + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Fix projects + eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputPath${graphInputPath}/project + --outputPath${graphOutputPath}/project + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Project + --isLookupUrl${isLookupUrl} + + + + + + + + yarn + cluster + Fix relations + eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + dhp-graph-mapper-${projectVersion}.jar + + --executor-cores=${sparkExecutorCoresForJoining} + --executor-memory=${sparkExecutorMemoryForJoining} + --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=7680 + --conf spark.network.timeout=${sparkNetworkTimeout} + + --inputPath${graphInputPath}/relation + --outputPath${graphOutputPath}/relation + --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Relation + --isLookupUrl${isLookupUrl} + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_fix_graph_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_fix_graph_parameters.json new file mode 100644 index 000000000..9cfed1e91 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_fix_graph_parameters.json @@ -0,0 +1,32 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false + }, + { + "paramName": "in", + "paramLongName": "inputPath", + "paramDescription": "the path to the graph data dump to read", + "paramRequired": true + }, + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the path to store the output graph", + "paramRequired": true + }, + { + "paramName": "isu", + "paramLongName": "isLookupUrl", + "paramDescription": "url to the ISLookup Service", + "paramRequired": true + }, + { + "paramName": "class", + "paramLongName": "graphTableClassName", + "paramDescription": "class name moelling the graph table", + "paramRequired": true + } +] From d9f33582c5cd10f4fc27973f4e62291aa096724b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 9 Jun 2020 17:20:40 +0200 Subject: [PATCH 50/88] WIP: graph cleaner implementation --- .../oa/graph/clean/CleanGraphProperties.java | 119 ++ .../dhp/oa/graph/clean/CleaningRule.java | 89 ++ .../dhp/oa/graph/fix/FixGraphProperties.java | 210 --- .../raw/GenerateEntitiesApplication.java | 5 +- .../raw/MigrateDbEntitiesApplication.java | 3 +- .../oa/graph/raw/common/OafMapperUtils.java | 4 + .../dhp/oa/graph/raw/common/Vocabulary.java | 45 +- .../oa/graph/raw/common/VocabularyGroup.java | 75 +- .../graph/{fix => clean}/config-default.xml | 0 .../dhp/oa/graph/{fix => clean}/workflow.xml | 90 +- ...json => input_clean_graph_parameters.json} | 0 .../xquery/load_vocabularies_synonyms.xquery | 6 + .../dhp/oa/graph/clean/CleaningRuleTest.java | 71 + .../eu/dnetlib/dhp/oa/graph/clean/result.json | 691 +++++++++ .../dnetlib/dhp/oa/graph/clean/synonyms.txt | 1279 +++++++++++++++++ .../eu/dnetlib/dhp/oa/graph/clean/terms.txt | 1044 ++++++++++++++ 16 files changed, 3454 insertions(+), 277 deletions(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphProperties.java create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java delete mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/fix/FixGraphProperties.java rename dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/{fix => clean}/config-default.xml (100%) rename dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/{fix => clean}/workflow.xml (85%) rename dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/{input_fix_graph_parameters.json => input_clean_graph_parameters.json} (100%) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies_synonyms.xquery create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphProperties.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphProperties.java new file mode 100644 index 000000000..86dca6d21 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphProperties.java @@ -0,0 +1,119 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.beans.Introspector; +import java.beans.PropertyDescriptor; +import java.lang.reflect.InvocationTargetException; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import scala.Predef; + +public class CleanGraphProperties { + + private static final Logger log = LoggerFactory.getLogger(CleanGraphProperties.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + CleanGraphProperties.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json")); + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String inputPath = parser.get("inputPath"); + log.info("inputPath: {}", inputPath); + + String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); + + String graphTableClassName = parser.get("graphTableClassName"); + log.info("graphTableClassName: {}", graphTableClassName); + + Class entityClazz = (Class) Class.forName(graphTableClassName); + + final ISLookUpService isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl); + final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookupService); + + SparkConf conf = new SparkConf(); + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + fixGraphTable(spark, vocs, inputPath, entityClazz, outputPath); + }); + } + + private static void fixGraphTable( + SparkSession spark, + VocabularyGroup vocs, + String inputPath, + Class clazz, + String outputPath) { + + CleaningRule rule = new CleaningRule<>(vocs); + + readTableFromPath(spark, inputPath, clazz) + .map(rule, Encoders.bean(clazz)) + .write() + .mode(SaveMode.Overwrite) + .parquet(outputPath); + } + + private static Dataset readTableFromPath( + SparkSession spark, String inputEntityPath, Class clazz) { + + log.info("Reading Graph table from: {}", inputEntityPath); + return spark + .read() + .textFile(inputEntityPath) + .map( + (MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), + Encoders.bean(clazz)); + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java new file mode 100644 index 000000000..c00c7b4d9 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java @@ -0,0 +1,89 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.beans.Introspector; +import java.beans.PropertyDescriptor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.util.*; + +import org.apache.spark.api.java.function.MapFunction; + +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.oaf.Oaf; +import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Qualifier; + +public class CleaningRule implements MapFunction { + + private VocabularyGroup vocabularies; + + public CleaningRule(VocabularyGroup vocabularies) { + this.vocabularies = vocabularies; + } + + @Override + public T call(T value) throws Exception { + + doClean(value); + + return value; + } + + private void doClean(Object o) { + if (Objects.isNull(o)) { + return; + } + + if (o instanceof Iterable) { + for (Object oi : (Iterable) o) { + doClean(oi); + } + } else { + + Class clazz = o.getClass(); + + if (clazz.isPrimitive() + || o instanceof Integer + || o instanceof Double + || o instanceof Float + || o instanceof Long + || o instanceof Boolean + || o instanceof String) { + return; + } else { + try { + for (Field field : getAllFields(new LinkedList<>(), clazz)) { + field.setAccessible(true); + Object value = field.get(o); + if (value instanceof Qualifier) { + Qualifier q = (Qualifier) value; + if (vocabularies.vocabularyExists(q.getSchemeid())) { + + field.set(o, vocabularies.lookup(q.getSchemeid(), q.getClassid())); + } + + } else { + doClean(value); + } + } + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new RuntimeException(e); + } + } + } + } + + private static List getAllFields(List fields, Class clazz) { + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + + final Class superclass = clazz.getSuperclass(); + if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { + getAllFields(fields, superclass); + } + + return fields; + } +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/fix/FixGraphProperties.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/fix/FixGraphProperties.java deleted file mode 100644 index bbcadde59..000000000 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/fix/FixGraphProperties.java +++ /dev/null @@ -1,210 +0,0 @@ -package eu.dnetlib.dhp.oa.graph.fix; - -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils; -import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; -import eu.dnetlib.dhp.schema.common.EntityType; -import eu.dnetlib.dhp.schema.common.ModelSupport; -import eu.dnetlib.dhp.schema.oaf.*; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import scala.Tuple2; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - -public class FixGraphProperties { - - private static final Logger log = LoggerFactory.getLogger(FixGraphProperties.class); - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - public static void main(String[] args) throws Exception { - - String jsonConfiguration = IOUtils - .toString( - FixGraphProperties.class - .getResourceAsStream( - "/eu/dnetlib/dhp/oa/graph/input_fix_graph_parameters.json")); - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - parser.parseArgument(args); - - Boolean isSparkSessionManaged = Optional - .ofNullable(parser.get("isSparkSessionManaged")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - - String inputPath = parser.get("inputPath"); - log.info("inputPath: {}", inputPath); - - String outputPath = parser.get("outputPath"); - log.info("outputPath: {}", outputPath); - - String isLookupUrl = parser.get("isLookupUrl"); - log.info("isLookupUrl: {}", isLookupUrl); - - String graphTableClassName = parser.get("graphTableClassName"); - log.info("graphTableClassName: {}", graphTableClassName); - - Class entityClazz = (Class) Class.forName(graphTableClassName); - - final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookupUrl); - - SparkConf conf = new SparkConf(); - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { - removeOutputDir(spark, outputPath); - fixGraphTable(spark, vocs, inputPath, entityClazz, outputPath); - }); - } - - private static void fixGraphTable( - SparkSession spark, - VocabularyGroup vocs, - String inputPath, - Class clazz, - String outputPath) { - - MapFunction fixFn = getFixingFunction(vocs, clazz); - - readTableFromPath(spark, inputPath, clazz) - .map(fixFn, Encoders.bean(clazz)) - .write() - .mode(SaveMode.Overwrite) - .parquet(outputPath); - } - - private static MapFunction getFixingFunction(VocabularyGroup vocs, Class clazz) { - - switch (clazz.getCanonicalName()) { - case "eu.dnetlib.dhp.schema.oaf.Publication": - case "eu.dnetlib.dhp.schema.oaf.Dataset": - case "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct": - case "eu.dnetlib.dhp.schema.oaf.Software": - return (MapFunction) value -> { - Result r = (Result) value; - - if (r.getLanguage() != null) { - r.setLanguage(vocs.getTermAsQualifier("dnet:languages", "und")); - } else { - r.setLanguage(vocs.getTermAsQualifier("dnet:languages", r.getLanguage().getClassid())); - } - if (r.getCountry() != null) { - r.setCountry( - r.getCountry() - .stream() - .filter(Objects::nonNull) - .map(c -> { - Qualifier q = vocs.getTermAsQualifier("dnet:countries", c.getClassid()); - Country cn = new Country(); - cn.setDataInfo(c.getDataInfo()); - cn.setClassid(q.getClassid()); - cn.setClassname(cn.getClassname()); - cn.setSchemeid("dnet:countries"); - cn.setSchemename("dnet:countries"); - return cn; - }) - .collect(Collectors.toList())); - } - - if (r.getSubject() != null) { - r.setSubject( - r.getSubject() - .stream() - .filter(Objects::nonNull) - .map(s -> { - if (s.getQualifier() == null || StringUtils.isBlank(s.getQualifier().getClassid())) { - s.setQualifier(vocs.getTermAsQualifier("dnet:subject_classification_typologies", "UNKNOWN")); - } - }) - .collect(Collectors.toList()) - ); - } - - if (r.getPublisher() != null && StringUtils.isBlank(r.getPublisher().getValue())) { - r.setPublisher(null); - } - if (r.getBestaccessright() == null) { - r.setBestaccessright(vocs.getTermAsQualifier("dnet:access_modes", "UNKNOWN")); - } - if (r.getInstance() != null) { - for(Instance i : r.getInstance()) { - if (i.getAccessright() == null) { - i.setAccessright(vocs.getTermAsQualifier("dnet:access_modes", "UNKNOWN")); - } - if (i.getInstancetype() != null) { - i.setInstancetype(vocs.getTermAsQualifier("dnet:publication_resource", i.getInstancetype().getClassid())); - } else { - i.setInstancetype(vocs.getTermAsQualifier("dnet:publication_resource", "0000")); - } - - - } - } - - return clazz.cast(r); - }; - case "eu.dnetlib.dhp.schema.oaf.Datasource": - return (MapFunction) value -> { - return value; - }; - case "eu.dnetlib.dhp.schema.oaf.Organization": - return (MapFunction) value -> { - Organization o = (Organization) value; - - if (o.getCountry() == null) { - o.setCountry(vocs.getTermAsQualifier("dnet:countries", "UNKNOWN")); - } else { - o.setCountry(vocs.getTermAsQualifier("dnet:countries", o.getCountry().getClassid())); - } - - return clazz.cast(o); - }; - case "eu.dnetlib.dhp.schema.oaf.Project": - return (MapFunction) value -> { - return value; - }; - case "eu.dnetlib.dhp.schema.oaf.Relation": - return (MapFunction) value -> { - return value; - }; - default: - throw new RuntimeException("unknown class: " + clazz.getCanonicalName()); - } - - } - - private static Dataset readTableFromPath( - SparkSession spark, String inputEntityPath, Class clazz) { - - log.info("Reading Graph table from: {}", inputEntityPath); - return spark - .read() - .textFile(inputEntityPath) - .map( - (MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), - Encoders.bean(clazz)); - } - - private static void removeOutputDir(SparkSession spark, String path) { - HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); - } - -} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java index 8e5ba9cd1..704f101ae 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java @@ -39,6 +39,8 @@ import eu.dnetlib.dhp.schema.oaf.Project; import eu.dnetlib.dhp.schema.oaf.Publication; import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.dhp.schema.oaf.Software; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; import scala.Tuple2; public class GenerateEntitiesApplication { @@ -71,7 +73,8 @@ public class GenerateEntitiesApplication { final String isLookupUrl = parser.get("isLookupUrl"); log.info("isLookupUrl: {}", isLookupUrl); - final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookupUrl); + final ISLookUpService isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl); + final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookupService); final SparkConf conf = new SparkConf(); runWithSparkSession(conf, isSparkSessionManaged, spark -> { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java index 653027039..89bf5c379 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java @@ -71,6 +71,7 @@ import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.dhp.schema.oaf.Software; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; public class MigrateDbEntitiesApplication extends AbstractMigrationApplication implements Closeable { @@ -151,7 +152,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i super(hdfsPath); this.dbClient = new DbClient(dbUrl, dbUser, dbPassword); this.lastUpdateTimestamp = new Date().getTime(); - this.vocs = VocabularyGroup.loadVocsFromIS(isLookupUrl); + this.vocs = VocabularyGroup.loadVocsFromIS(ISLookupClientFactory.getLookUpService(isLookupUrl)); } public void execute(final String sqlFile, final Function> producer) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java index 9beed2837..58f068943 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java @@ -60,6 +60,10 @@ public class OafMapperUtils { .collect(Collectors.toList()); } + public static Qualifier unknown(final String schemeid, final String schemename) { + return qualifier("UNKNOWN", "Unknown", schemeid, schemename); + } + public static Qualifier qualifier( final String classid, final String classname, diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java index c955ee640..ffa64861c 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java @@ -4,14 +4,29 @@ package eu.dnetlib.dhp.oa.graph.raw.common; import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import java.util.Optional; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.collect.Maps; + +import eu.dnetlib.dhp.schema.oaf.Qualifier; public class Vocabulary implements Serializable { private final String id; private final String name; + /** + * Code to Term mappings for this Vocabulary. + */ private final Map terms = new HashMap<>(); + /** + * Synonym to Code mappings for this Vocabulary. + */ + private final Map synonyms = Maps.newHashMap(); + public Vocabulary(final String id, final String name) { this.id = id; this.name = name; @@ -30,7 +45,7 @@ public class Vocabulary implements Serializable { } public VocabularyTerm getTerm(final String id) { - return terms.get(id.toLowerCase()); + return Optional.ofNullable(id).map(s -> s.toLowerCase()).map(s -> terms.get(s)).orElse(null); } protected void addTerm(final String id, final String name) { @@ -40,4 +55,32 @@ public class Vocabulary implements Serializable { protected boolean termExists(final String id) { return terms.containsKey(id.toLowerCase()); } + + protected void addSynonym(final String syn, final String termCode) { + synonyms.put(syn, termCode.toLowerCase()); + } + + public VocabularyTerm getTermBySynonym(final String syn) { + return getTerm(synonyms.get(syn)); + } + + public Qualifier getTermAsQualifier(final String termId) { + if (StringUtils.isBlank(termId)) { + return OafMapperUtils.unknown(getId(), getName()); + } else if (termExists(termId)) { + final VocabularyTerm t = getTerm(termId); + return OafMapperUtils.qualifier(t.getId(), t.getName(), getId(), getName()); + } else { + return OafMapperUtils.qualifier(termId, termId, getId(), getName()); + } + } + + public Qualifier getSynonymAsQualifier(final String syn) { + return Optional + .ofNullable(getTermBySynonym(syn)) + .map(term -> getTermAsQualifier(term.getId())) + .orElse(null); + // .orElse(OafMapperUtils.unknown(getId(), getName())); + } + } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java index 43ed7f2d9..6af80683e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java @@ -1,33 +1,40 @@ package eu.dnetlib.dhp.oa.graph.raw.common; -import java.io.IOException; import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import eu.dnetlib.dhp.oa.graph.raw.GenerateEntitiesApplication; import eu.dnetlib.dhp.schema.oaf.Qualifier; -import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; public class VocabularyGroup implements Serializable { - public static VocabularyGroup loadVocsFromIS(final String isLookupUrl) throws IOException, ISLookUpException { - final ISLookUpService isLookUpService = ISLookupClientFactory.getLookUpService(isLookupUrl); + public static final String VOCABULARIES_XQUERY = "for $x in collection(' /db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') \n" + + + "let $vocid := $x//VOCABULARY_NAME/@code\n" + + "let $vocname := $x//VOCABULARY_NAME/text()\n" + + "for $term in ($x//TERM)\n" + + "return concat($vocid,' @=@ ',$vocname,' @=@ ',$term/@code,' @=@ ',$term/@english_name)"; - final String xquery = IOUtils - .toString( - GenerateEntitiesApplication.class - .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies.xquery")); + public static final String VOCABULARY_SYNONYMS_XQUERY = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')\n" + + + "let $vocid := $x//VOCABULARY_NAME/@code\n" + + "let $vocname := $x//VOCABULARY_NAME/text()\n" + + "for $term in ($x//TERM)\n" + + "for $syn in ($term//SYNONYM/@term)\n" + + "return concat($vocid,' @=@ ',$term/@code,' @=@ ', $syn)\n"; + + public static VocabularyGroup loadVocsFromIS(ISLookUpService isLookUpService) throws ISLookUpException { final VocabularyGroup vocs = new VocabularyGroup(); - for (final String s : isLookUpService.quickSearchProfile(xquery)) { + for (final String s : isLookUpService.quickSearchProfile(VOCABULARIES_XQUERY)) { final String[] arr = s.split("@=@"); if (arr.length == 4) { final String vocId = arr[0].trim(); @@ -40,6 +47,19 @@ public class VocabularyGroup implements Serializable { } vocs.addTerm(vocId, termId, termName); + vocs.addSynonyms(vocId, termId, termId); + } + } + + for (final String s : isLookUpService.quickSearchProfile(VOCABULARY_SYNONYMS_XQUERY)) { + final String[] arr = s.split("@=@"); + if (arr.length == 3) { + final String vocId = arr[0].trim(); + final String termId = arr[1].trim(); + final String syn = arr[2].trim(); + + vocs.addSynonyms(vocId, termId, syn); + vocs.addSynonyms(vocId, termId, termId); } } @@ -66,16 +86,21 @@ public class VocabularyGroup implements Serializable { } } + public Qualifier lookup(String vocId, String id) { + return Optional + .ofNullable(getSynonymAsQualifier(vocId, id)) + .orElse(getTermAsQualifier(vocId, id)); + } + public Qualifier getTermAsQualifier(final String vocId, final String id) { - if (StringUtils.isBlank(id)) { - return OafMapperUtils.qualifier("UNKNOWN", "UNKNOWN", vocId, vocId); - } else if (termExists(vocId, id)) { - final Vocabulary v = vocs.get(vocId.toLowerCase()); - final VocabularyTerm t = v.getTerm(id); - return OafMapperUtils.qualifier(t.getId(), t.getName(), v.getId(), v.getName()); - } else { - return OafMapperUtils.qualifier(id, id, vocId, vocId); + return vocs.get(vocId.toLowerCase()).getTermAsQualifier(id); + } + + public Qualifier getSynonymAsQualifier(final String vocId, final String syn) { + if (StringUtils.isBlank(vocId)) { + return OafMapperUtils.unknown("", ""); } + return vocs.get(vocId.toLowerCase()).getSynonymAsQualifier(syn); } public boolean termExists(final String vocId, final String id) { @@ -86,4 +111,16 @@ public class VocabularyGroup implements Serializable { return vocs.containsKey(vocId.toLowerCase()); } + private void addSynonyms(final String vocId, final String termId, final String syn) { + String id = Optional + .ofNullable(vocId) + .map(s -> s.toLowerCase()) + .orElseThrow( + () -> new IllegalArgumentException(String.format("empty vocabulary id for [term:%s, synonym:%s]"))); + Optional + .ofNullable(vocs.get(id)) + .orElseThrow(() -> new IllegalArgumentException("missing vocabulary id: " + vocId)) + .addSynonym(syn, termId); + } + } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/config-default.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/config-default.xml similarity index 100% rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/config-default.xml rename to dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/config-default.xml diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/workflow.xml similarity index 85% rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/workflow.xml rename to dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/workflow.xml index e93f58279..d152448af 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/fix/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/workflow.xml @@ -1,4 +1,4 @@ - + @@ -7,7 +7,7 @@ graphOutputPath - the target path to store fixed graph + the target path to store cleaned graph isLookupUrl @@ -50,29 +50,29 @@ - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - - - - - - - - - + + + + + + + + + - + yarn cluster - Fix publications - eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + Clean publications + eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties dhp-graph-mapper-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -90,16 +90,16 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication --isLookupUrl${isLookupUrl} - + - + yarn cluster - Fix datasets - eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + Clean datasets + eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties dhp-graph-mapper-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -117,16 +117,16 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Dataset --isLookupUrl${isLookupUrl} - + - + yarn cluster - Fix otherresearchproducts - eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + Clean otherresearchproducts + eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties dhp-graph-mapper-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -144,16 +144,16 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct --isLookupUrl${isLookupUrl} - + - + yarn cluster - Fix softwares - eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + Clean softwares + eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties dhp-graph-mapper-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -171,16 +171,16 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Software --isLookupUrl${isLookupUrl} - + - + yarn cluster - Fix datasources - eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + Clean datasources + eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties dhp-graph-mapper-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -198,16 +198,16 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Datasource --isLookupUrl${isLookupUrl} - + - + yarn cluster - Fix organizations - eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + Clean organizations + eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties dhp-graph-mapper-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -225,16 +225,16 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Organization --isLookupUrl${isLookupUrl} - + - + yarn cluster - Fix projects - eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + Clean projects + eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties dhp-graph-mapper-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -252,16 +252,16 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Project --isLookupUrl${isLookupUrl} - + - + yarn cluster - Fix relations - eu.dnetlib.dhp.oa.graph.fix.FixGraphProperties + Clean relations + eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties dhp-graph-mapper-${projectVersion}.jar --executor-cores=${sparkExecutorCoresForJoining} @@ -279,11 +279,11 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Relation --isLookupUrl${isLookupUrl} - + - + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_fix_graph_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json similarity index 100% rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_fix_graph_parameters.json rename to dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies_synonyms.xquery b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies_synonyms.xquery new file mode 100644 index 000000000..f4f8cb45d --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/xquery/load_vocabularies_synonyms.xquery @@ -0,0 +1,6 @@ +for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') +let $vocid := $x//VOCABULARY_NAME/@code +let $vocname := $x//VOCABULARY_NAME/text() + for $term in ($x//TERM) + for $syn in ($term//SYNONYM/@term) + return concat($vocid,' @=@ ',$term/@code,' @=@ ', $syn) diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java new file mode 100644 index 000000000..fab3c0c01 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java @@ -0,0 +1,71 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import static org.mockito.Mockito.lenient; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; + +@ExtendWith(MockitoExtension.class) +public class CleaningRuleTest { + + public static final ObjectMapper MAPPER = new ObjectMapper(); + @Mock + private ISLookUpService isLookUpService; + + private VocabularyGroup vocabularies; + + private CleaningRule cleaningRule; + + @BeforeEach + public void setUp() throws ISLookUpException, IOException { + lenient().when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARIES_XQUERY)).thenReturn(vocs()); + lenient() + .when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARY_SYNONYMS_XQUERY)) + .thenReturn(synonyms()); + + vocabularies = VocabularyGroup.loadVocsFromIS(isLookUpService); + cleaningRule = new CleaningRule(vocabularies); + } + + @Test + public void testCleaning() throws Exception { + + String json = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/result.json")); + Publication p_in = MAPPER.readValue(json, Publication.class); + + Publication p_out = cleaningRule.call(p_in); + + Assertions.assertNotNull(p_out); + + // TODO add more assertions to verity the cleaned values + System.out.println(MAPPER.writeValueAsString(p_out)); + + } + + private List vocs() throws IOException { + return IOUtils + .readLines(CleaningRuleTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/terms.txt")); + } + + private List synonyms() throws IOException { + return IOUtils + .readLines(CleaningRuleTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt")); + } +} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json new file mode 100644 index 000000000..c45544b40 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json @@ -0,0 +1,691 @@ +{ + "author": [ + { + "affiliation": [ + ], + "fullname": "Brien, Tom", + "name": "Tom", + "pid": [ + ], + "rank": 1, + "surname": "Brien" + }, + { + "affiliation": [ + ], + "fullname": "Ade, Peter", + "name": "Peter", + "pid": [ + ], + "rank": 2, + "surname": "Ade" + }, + { + "affiliation": [ + ], + "fullname": "Barry, Peter S.", + "name": "Peter S.", + "pid": [ + ], + "rank": 3, + "surname": "Barry" + }, + { + "affiliation": [ + ], + "fullname": "Dunscombe, Chris J.", + "name": "Chris J.", + "pid": [ + ], + "rank": 4, + "surname": "Dunscombe" + }, + { + "affiliation": [ + ], + "fullname": "Leadley, David R.", + "name": "David R.", + "pid": [ + ], + "rank": 5, + "surname": "Leadley" + }, + { + "affiliation": [ + ], + "fullname": "Morozov, Dmitry V.", + "name": "Dmitry V.", + "pid": [ + ], + "rank": 6, + "surname": "Morozov" + }, + { + "affiliation": [ + ], + "fullname": "Myronov, Maksym", + "name": "Maksym", + "pid": [ + ], + "rank": 7, + "surname": "Myronov" + }, + { + "affiliation": [ + ], + "fullname": "Parker, Evan", + "name": "Evan", + "pid": [ + ], + "rank": 8, + "surname": "Parker" + }, + { + "affiliation": [ + ], + "fullname": "Prest, Martin J.", + "name": "Martin J.", + "pid": [ + ], + "rank": 9, + "surname": "Prest" + }, + { + "affiliation": [ + ], + "fullname": "Prunnila, Mika", + "name": "Mika", + "pid": [ + ], + "rank": 10, + "surname": "Prunnila" + }, + { + "affiliation": [ + ], + "fullname": "Sudiwala, Rashmi V.", + "name": "Rashmi V.", + "pid": [ + ], + "rank": 11, + "surname": "Sudiwala" + }, + { + "affiliation": [ + ], + "fullname": "Whall, Terry E.", + "name": "Terry E.", + "pid": [ + ], + "rank": 12, + "surname": "Whall" + }, + { + "affiliation": [ + ], + "fullname": "Mauskopf", + "name": "", + "pid": [ + ], + "rank": 13, + "surname": "" + }, + { + "affiliation": [ + ], + "fullname": " P. D. ", + "name": "", + "pid": [ + ], + "rank": 14, + "surname": "" + } + ], + "bestaccessright": { + "classid": "CLOSED", + "classname": "Closed Access", + "schemeid": "dnet:access_modes", + "schemename": "dnet:access_modes" + }, + "collectedfrom": [ + { + "key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747", + "value": "VIRTA" + } + ], + "context": [ + ], + "contributor": [ + ], + "country": [ + ], + "coverage": [ + ], + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "value": "2016-01-01" + }, + "dateofcollection": "", + "dateoftransformation": "2020-04-22T12:34:08.009Z", + "description": [ + ], + "externalReference": [ + ], + "extraInfo": [ + ], + "format": [ + ], + "fulltext": [ + ], + "id": "50|CSC_________::2250a70c903c6ac6e4c01438259e9375", + "instance": [ + { + "accessright": { + "classid": "CLOSED", + "classname": "Closed Access", + "schemeid": "dnet:access_modes", + "schemename": "dnet:access_modes" + }, + "collectedfrom": { + "key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747", + "value": "VIRTA" + }, + "dateofacceptance": { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "value": "2016-01-01" + }, + "distributionlocation": "", + "hostedby": { + "key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747", + "value": "VIRTA" + }, + "instancetype": { + "classid": "Comentario", + "classname": "Comentario", + "schemeid": "dnet:publication_resource", + "schemename": "dnet:publication_resource" + }, + "url": [ + "http://juuli.fi/Record/0275158616", + "http://dx.doi.org/10.1007/s109090161569x" + ] + } + ], + "journal": { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "edition": "", + "ep": " 7", + "iss": "9 March", + "issnLinking": "", + "issnOnline": "", + "issnPrinted": "0022-2291", + "name": "Journal of Low Temperature Physics - Early Acces", + "sp": "1 ", + "vol": "" + }, + "language": { + "classid": "en", + "classname": "en", + "schemeid": "dnet:languages", + "schemename": "dnet:languages" + }, + "lastupdatetimestamp": 1591283286319, + "oaiprovenance": { + "originDescription": { + "altered": true, + "baseURL": "https%3A%2F%2Fvirta-jtp.csc.fi%2Fapi%2Fcerif", + "datestamp": "2019-07-30", + "harvestDate": "2020-04-22T11:04:38.685Z", + "identifier": "oai:virta-jtp.csc.fi:Publications/0275158616", + "metadataNamespace": "" + } + }, + "originalId": [ + "CSC_________::2250a70c903c6ac6e4c01438259e9375" + ], + "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "doi", + "classname": "doi", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "10.1007/s109090161569x" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "doi", + "classname": "doi", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "10.1007/s109090161569x" + } + ], + "relevantdate": [ + ], + "resourcetype": { + "classid": "0001", + "classname": "0001", + "schemeid": "dnet:dataCite_resource", + "schemename": "dnet:dataCite_resource" + }, + "resulttype": { + "classid": "publication", + "classname": "publication", + "schemeid": "dnet:result_typologies", + "schemename": "dnet:result_typologies" + }, + "source": [ + ], + "subject": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "ta213" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "infrared detectors" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "lens antennas" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "silicon" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "slot antennas" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "strained silicon" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "cold electron bolometers" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "doped silicon" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "measure noise" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "noise equivalent power" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "optical characterisation" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "optical response" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "photon noise" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "", + "classname": "", + "schemeid": "", + "schemename": "" + }, + "value": "silicon absorbers" + } + ], + "title": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "main title", + "classname": "main title", + "schemeid": "dnet:dataCite_title", + "schemename": "dnet:dataCite_title" + }, + "value": "Optical response of strained- and unstrained-silicon cold-electron bolometers" + } + ] +} \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt new file mode 100644 index 000000000..10eba3e86 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt @@ -0,0 +1,1279 @@ +dnet:countries @=@ AF @=@ AFG +dnet:countries @=@ AF @=@ Afghanistan +dnet:countries @=@ AD @=@ Andorra +dnet:countries @=@ AO @=@ Angola +dnet:countries @=@ AR @=@ ARG +dnet:countries @=@ AR @=@ Argentina +dnet:countries @=@ AU @=@ AUS +dnet:countries @=@ AU @=@ Australia +dnet:countries @=@ AT @=@ AUT +dnet:countries @=@ AT @=@ Austria +dnet:countries @=@ AZ @=@ AZE +dnet:countries @=@ BD @=@ Bangladesh +dnet:countries @=@ BY @=@ Belarus +dnet:countries @=@ BE @=@ BEL +dnet:countries @=@ BE @=@ Belgium +dnet:countries @=@ BJ @=@ BEN +dnet:countries @=@ BO @=@ Bolivia, Plurinational State of +dnet:countries @=@ BA @=@ BIH +dnet:countries @=@ BA @=@ Bosnia-Hercegovina +dnet:countries @=@ BR @=@ BRA +dnet:countries @=@ BR @=@ Brazil +dnet:countries @=@ BG @=@ Bulgaria +dnet:countries @=@ BF @=@ BFA +dnet:countries @=@ KH @=@ Campuchea +dnet:countries @=@ CM @=@ CMR +dnet:countries @=@ CA @=@ CAN +dnet:countries @=@ CA @=@ Canada +dnet:countries @=@ CV @=@ Cape Verde +dnet:countries @=@ CL @=@ CHL +dnet:countries @=@ CL @=@ Chile +dnet:countries @=@ CN @=@ CHN +dnet:countries @=@ CN @=@ China +dnet:countries @=@ CO @=@ COL +dnet:countries @=@ CO @=@ Colombia +dnet:countries @=@ CG @=@ Congo, Republic +dnet:countries @=@ CD @=@ Congo, the Democratic Republic of the +dnet:countries @=@ CD @=@ Congo Democratic Republic (formerly Zaire) +dnet:countries @=@ CD @=@ Congo Democratic Republic +dnet:countries @=@ CD @=@ Zaire +dnet:countries @=@ CR @=@ CRI +dnet:countries @=@ CI @=@ CIV +dnet:countries @=@ CI @=@ Ivory Coast +dnet:countries @=@ HR @=@ Croatia +dnet:countries @=@ HR @=@ HRV +dnet:countries @=@ CY @=@ CYP +dnet:countries @=@ CY @=@ Cyprus +dnet:countries @=@ CZ @=@ CZE +dnet:countries @=@ CZ @=@ Czech Republic +dnet:countries @=@ CZ @=@ Czechoslovakia +dnet:countries @=@ DK @=@ DNK +dnet:countries @=@ DK @=@ Denmark +dnet:countries @=@ EC @=@ Ecuador +dnet:countries @=@ EG @=@ EGY +dnet:countries @=@ EG @=@ Egypt +dnet:countries @=@ SV @=@ SLV +dnet:countries @=@ EE @=@ EST +dnet:countries @=@ EE @=@ Estonia +dnet:countries @=@ ET @=@ ETH +dnet:countries @=@ EU @=@ EEC +dnet:countries @=@ FI @=@ FIN +dnet:countries @=@ FI @=@ Finland +dnet:countries @=@ MK @=@ Macedonia, the Former Yugoslav Republic Of +dnet:countries @=@ MK @=@ Macedonia +dnet:countries @=@ FR @=@ FRA +dnet:countries @=@ FR @=@ France +dnet:countries @=@ PF @=@ French Polynesia +dnet:countries @=@ PF @=@ PYF +dnet:countries @=@ TF @=@ French Southern Territories +dnet:countries @=@ GE @=@ Georgia +dnet:countries @=@ DE @=@ DEU +dnet:countries @=@ DE @=@ Germany +dnet:countries @=@ DE @=@ Germany, Berlin +dnet:countries @=@ GH @=@ GHA +dnet:countries @=@ GR @=@ EL +dnet:countries @=@ GR @=@ GRC +dnet:countries @=@ GL @=@ GRL +dnet:countries @=@ GN @=@ Guinea +dnet:countries @=@ GW @=@ Guinea-Bissau +dnet:countries @=@ VA @=@ Vatican State +dnet:countries @=@ HK @=@ HKG +dnet:countries @=@ HK @=@ Hong Kong +dnet:countries @=@ HK @=@ Hongkong +dnet:countries @=@ HU @=@ HUN +dnet:countries @=@ HU @=@ Hungary +dnet:countries @=@ IS @=@ ISL +dnet:countries @=@ IN @=@ IND +dnet:countries @=@ IN @=@ India +dnet:countries @=@ ID @=@ IDN +dnet:countries @=@ ID @=@ Indonesia +dnet:countries @=@ IR @=@ Iran +dnet:countries @=@ IR @=@ Iran, Islamic Republic of +dnet:countries @=@ IE @=@ IRL +dnet:countries @=@ IE @=@ Ireland +dnet:countries @=@ IL @=@ ISR +dnet:countries @=@ IL @=@ Israel +dnet:countries @=@ IT @=@ ITA +dnet:countries @=@ IT @=@ Italy +dnet:countries @=@ JM @=@ Jamaica +dnet:countries @=@ JP @=@ JPN +dnet:countries @=@ JP @=@ Japan +dnet:countries @=@ KZ @=@ KAZ +dnet:countries @=@ KZ @=@ Kazakistan +dnet:countries @=@ KZ @=@ Kazakstan +dnet:countries @=@ KE @=@ KEN +dnet:countries @=@ KE @=@ Kenya +dnet:countries @=@ KR @=@ KOR +dnet:countries @=@ KR @=@ Korea, Republic of +dnet:countries @=@ KR @=@ Korean Republic (South Korea) +dnet:countries @=@ KP @=@ PRK +dnet:countries @=@ LY @=@ Libya +dnet:countries @=@ LT @=@ LTU +dnet:countries @=@ LU @=@ LUX +dnet:countries @=@ LU @=@ Luxembourg +dnet:countries @=@ MO @=@ Macao +dnet:countries @=@ MG @=@ Madagascar +dnet:countries @=@ MY @=@ Malaysia +dnet:countries @=@ ML @=@ Mali +dnet:countries @=@ MT @=@ Malta +dnet:countries @=@ MU @=@ Mauritius +dnet:countries @=@ MX @=@ MEX +dnet:countries @=@ MX @=@ Mexico +dnet:countries @=@ FM @=@ Micronesia +dnet:countries @=@ MD @=@ Moldova +dnet:countries @=@ MD @=@ Moldova, Republic of +dnet:countries @=@ MN @=@ Mongolia +dnet:countries @=@ MA @=@ Morocco +dnet:countries @=@ MZ @=@ Mozambique +dnet:countries @=@ NA @=@ NAM +dnet:countries @=@ NL @=@ NLD +dnet:countries @=@ NL @=@ Netherlands +dnet:countries @=@ AN @=@ Netherlands Antilles +dnet:countries @=@ NZ @=@ NZL +dnet:countries @=@ NZ @=@ New Zealand +dnet:countries @=@ NO @=@ NOR +dnet:countries @=@ NO @=@ Norway +dnet:countries @=@ OC @=@ Australasia +dnet:countries @=@ OM @=@ Oman +dnet:countries @=@ PK @=@ PAK +dnet:countries @=@ PK @=@ Pakistan +dnet:countries @=@ PS @=@ Palestine, State of +dnet:countries @=@ PS @=@ Palestinian Territory, Occupied +dnet:countries @=@ PA @=@ PAN +dnet:countries @=@ PA @=@ Panama +dnet:countries @=@ PG @=@ PapuaNew Guinea +dnet:countries @=@ PE @=@ PER +dnet:countries @=@ PH @=@ PHL +dnet:countries @=@ PH @=@ Philippines +dnet:countries @=@ PL @=@ POL +dnet:countries @=@ PL @=@ Poland +dnet:countries @=@ PT @=@ PRT +dnet:countries @=@ PT @=@ Portugal +dnet:countries @=@ PR @=@ Puerto Rico +dnet:countries @=@ RO @=@ ROU +dnet:countries @=@ RO @=@ Romania +dnet:countries @=@ RU @=@ RUS +dnet:countries @=@ RU @=@ Russia +dnet:countries @=@ RU @=@ Russian Federation +dnet:countries @=@ RE @=@ Réunion +dnet:countries @=@ KN @=@ Saint Kitts And Nevis +dnet:countries @=@ SA @=@ Saudi Arabia +dnet:countries @=@ SN @=@ SEN +dnet:countries @=@ SG @=@ SGP +dnet:countries @=@ SG @=@ Singapore +dnet:countries @=@ SK @=@ SVK +dnet:countries @=@ SI @=@ SVN +dnet:countries @=@ SI @=@ Slovenia +dnet:countries @=@ ZA @=@ South Africa +dnet:countries @=@ ZA @=@ ZAF +dnet:countries @=@ ES @=@ ESP +dnet:countries @=@ ES @=@ Spain +dnet:countries @=@ LK @=@ Sri Lanka +dnet:countries @=@ LK @=@ LKA +dnet:countries @=@ SR @=@ Suriname +dnet:countries @=@ SE @=@ SWE +dnet:countries @=@ SE @=@ Sweden +dnet:countries @=@ CH @=@ CHE +dnet:countries @=@ CH @=@ Switzerland +dnet:countries @=@ SY @=@ Syria +dnet:countries @=@ ST @=@ Sao Tome and Principe +dnet:countries @=@ TW @=@ TWN +dnet:countries @=@ TW @=@ Taiwan, Province of China +dnet:countries @=@ TZ @=@ Tanzania +dnet:countries @=@ TZ @=@ Tanzania, United Republic of +dnet:countries @=@ TH @=@ THA +dnet:countries @=@ TH @=@ Thailand +dnet:countries @=@ TL @=@ East Timor +dnet:countries @=@ TN @=@ TUN +dnet:countries @=@ TN @=@ Tunisia +dnet:countries @=@ TR @=@ TUR +dnet:countries @=@ TR @=@ Turkey +dnet:countries @=@ UNKNOWN @=@ AAA +dnet:countries @=@ UNKNOWN @=@ [Unknown] +dnet:countries @=@ UNKNOWN @=@ _? +dnet:countries @=@ UA @=@ UKR +dnet:countries @=@ UA @=@ Ukraine +dnet:countries @=@ AE @=@ United Arab Emirates +dnet:countries @=@ GB @=@ GBR +dnet:countries @=@ GB @=@ UK +dnet:countries @=@ GB @=@ United Kingdom +dnet:countries @=@ GB @=@ Great Britain +dnet:countries @=@ GB @=@ Great Britain and Northern Ireland +dnet:countries @=@ US @=@ USA +dnet:countries @=@ US @=@ United States +dnet:countries @=@ US @=@ United States of America +dnet:countries @=@ UY @=@ Uruguay +dnet:countries @=@ UZ @=@ Uzbekistan +dnet:countries @=@ VE @=@ Venezuela, Bolivarian Republic of +dnet:countries @=@ VN @=@ Vietnam +dnet:countries @=@ VG @=@ British Virgin Islands +dnet:countries @=@ YU @=@ Jugoslavia +dnet:countries @=@ YU @=@ Yugoslavia +dnet:countries @=@ ZW @=@ ABW +dnet:datasourceCompatibilityLevel @=@ openaire-pub_4.0 @=@ openaire4.0 +dnet:access_modes @=@ CLOSED @=@ http://purl.org/coar/access_right/c_14cb +dnet:access_modes @=@ CLOSED @=@ info:eu-repo/semantics/closedAccess +dnet:access_modes @=@ EMBARGO @=@ http://purl.org/coar/access_right/c_f1cf +dnet:access_modes @=@ EMBARGO @=@ info:eu-repo/semantics/embargoedAccess +dnet:access_modes @=@ OPEN @=@ Creative Commons License [CC BY-NC-ND] http://creativecommons.org/licenses/by-nc-nd/3.0/de/ +dnet:access_modes @=@ OPEN @=@ Creative commons +dnet:access_modes @=@ OPEN @=@ Open Access +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-nc-nd/3.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-nc/3.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-sa/3.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-sa/4.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by/3.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by/3.0/us/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by/4.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/publicdomain/zero/1.0/ +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/publicdomain/zero/1.0/ & http://www.canadensys.net/norms +dnet:access_modes @=@ OPEN @=@ http://purl.org/coar/access_right/c_abf2 +dnet:access_modes @=@ OPEN @=@ info:eu-repo/semantics/openAccess +dnet:access_modes @=@ OPEN @=@ openAccess +dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-nc-nd/2.0/uk/ +dnet:access_modes @=@ RESTRICTED @=@ http://purl.org/coar/access_right/c_16ec +dnet:access_modes @=@ RESTRICTED @=@ info:eu-repo/semantics/restrictedAccess +dnet:result_typologies @=@ dataset @=@ 0021 +dnet:result_typologies @=@ dataset @=@ 0024 +dnet:result_typologies @=@ dataset @=@ 0025 +dnet:result_typologies @=@ dataset @=@ 0030 +dnet:result_typologies @=@ dataset @=@ 0033 +dnet:result_typologies @=@ dataset @=@ 0037 +dnet:result_typologies @=@ dataset @=@ 0039 +dnet:result_typologies @=@ other @=@ 0000 +dnet:result_typologies @=@ other @=@ 0010 +dnet:result_typologies @=@ other @=@ 0018 +dnet:result_typologies @=@ other @=@ 0020 +dnet:result_typologies @=@ other @=@ 0022 +dnet:result_typologies @=@ other @=@ 0023 +dnet:result_typologies @=@ other @=@ 0026 +dnet:result_typologies @=@ other @=@ 0027 +dnet:result_typologies @=@ other @=@ 0028 +dnet:result_typologies @=@ publication @=@ 0001 +dnet:result_typologies @=@ publication @=@ 0002 +dnet:result_typologies @=@ publication @=@ 0004 +dnet:result_typologies @=@ publication @=@ 0005 +dnet:result_typologies @=@ publication @=@ 0006 +dnet:result_typologies @=@ publication @=@ 0007 +dnet:result_typologies @=@ publication @=@ 0008 +dnet:result_typologies @=@ publication @=@ 0009 +dnet:result_typologies @=@ publication @=@ 0011 +dnet:result_typologies @=@ publication @=@ 0012 +dnet:result_typologies @=@ publication @=@ 0013 +dnet:result_typologies @=@ publication @=@ 0014 +dnet:result_typologies @=@ publication @=@ 0015 +dnet:result_typologies @=@ publication @=@ 0016 +dnet:result_typologies @=@ publication @=@ 0017 +dnet:result_typologies @=@ publication @=@ 0019 +dnet:result_typologies @=@ publication @=@ 0031 +dnet:result_typologies @=@ publication @=@ 0032 +dnet:result_typologies @=@ publication @=@ 0034 +dnet:result_typologies @=@ publication @=@ 0035 +dnet:result_typologies @=@ publication @=@ 0036 +dnet:result_typologies @=@ publication @=@ 0038 +dnet:result_typologies @=@ publication @=@ 0044 +dnet:result_typologies @=@ publication @=@ 0045 +dnet:result_typologies @=@ software @=@ 0029 +dnet:result_typologies @=@ software @=@ 0040 +dnet:languages @=@ abk @=@ ab +dnet:languages @=@ aar @=@ aa +dnet:languages @=@ afr @=@ af +dnet:languages @=@ alb/sqi @=@ sq +dnet:languages @=@ amh @=@ am +dnet:languages @=@ ara @=@ ar +dnet:languages @=@ arm/hye @=@ hy +dnet:languages @=@ asm @=@ as +dnet:languages @=@ ina @=@ ia +dnet:languages @=@ aym @=@ ay +dnet:languages @=@ aze @=@ az +dnet:languages @=@ bak @=@ ba +dnet:languages @=@ baq/eus @=@ eu +dnet:languages @=@ bel @=@ be +dnet:languages @=@ ben @=@ bn +dnet:languages @=@ bih @=@ bh +dnet:languages @=@ bis @=@ bi +dnet:languages @=@ bre @=@ br +dnet:languages @=@ bul @=@ bg +dnet:languages @=@ bur/mya @=@ my +dnet:languages @=@ cat @=@ ca +dnet:languages @=@ chi/zho @=@ zh +dnet:languages @=@ cos @=@ co +dnet:languages @=@ hrv @=@ hr +dnet:languages @=@ hrv @=@ hr +dnet:languages @=@ hrv @=@ scr/hrv +dnet:languages @=@ ces/cze @=@ cs +dnet:languages @=@ dan @=@ da +dnet:languages @=@ dut/nld @=@ dut/nla +dnet:languages @=@ dut/nld @=@ dutdut +dnet:languages @=@ dut/nld @=@ nl +dnet:languages @=@ dut/nld @=@ nl_be +dnet:languages @=@ dut/nld @=@ nl_nl +dnet:languages @=@ dut/nld @=@ nld +dnet:languages @=@ dzo @=@ dz +dnet:languages @=@ eng @=@ en +dnet:languages @=@ eng @=@ en_au +dnet:languages @=@ eng @=@ en_en +dnet:languages @=@ eng @=@ en_gb +dnet:languages @=@ eng @=@ en_nz +dnet:languages @=@ eng @=@ en_us +dnet:languages @=@ eng @=@ english +dnet:languages @=@ epo @=@ eo +dnet:languages @=@ est @=@ et +dnet:languages @=@ fao @=@ fo +dnet:languages @=@ fij @=@ fj +dnet:languages @=@ fin @=@ fi +dnet:languages @=@ fra/fre @=@ fr +dnet:languages @=@ fra/fre @=@ fr_be +dnet:languages @=@ fra/fre @=@ fr_fr +dnet:languages @=@ fra/fre @=@ fre/fra +dnet:languages @=@ fry @=@ fy +dnet:languages @=@ glg @=@ gl +dnet:languages @=@ geo/kat @=@ ka +dnet:languages @=@ deu/ger @=@ de +dnet:languages @=@ deu/ger @=@ ger/deu +dnet:languages @=@ deu/ger @=@ german +dnet:languages @=@ deu/ger @=@ de_DE +dnet:languages @=@ ell/gre @=@ el +dnet:languages @=@ ell/gre @=@ gr +dnet:languages @=@ kal @=@ kl +dnet:languages @=@ grn @=@ gn +dnet:languages @=@ guj @=@ gu +dnet:languages @=@ hau @=@ ha +dnet:languages @=@ heb @=@ he +dnet:languages @=@ hin @=@ hi +dnet:languages @=@ hun @=@ hu +dnet:languages @=@ ice/isl @=@ is +dnet:languages @=@ ine @=@ - +dnet:languages @=@ ind @=@ id +dnet:languages @=@ iku @=@ iu +dnet:languages @=@ ipk @=@ ik +dnet:languages @=@ gai/iri @=@ ga +dnet:languages @=@ gai/iri @=@ gle +dnet:languages @=@ ita @=@ it +dnet:languages @=@ jpn @=@ ja +dnet:languages @=@ jav @=@ jv +dnet:languages @=@ jav @=@ jv/jw +dnet:languages @=@ jav @=@ jw +dnet:languages @=@ kan @=@ kn +dnet:languages @=@ kas @=@ ks +dnet:languages @=@ kaz @=@ kk +dnet:languages @=@ khm @=@ km +dnet:languages @=@ kin @=@ rw +dnet:languages @=@ kir @=@ ky +dnet:languages @=@ kor @=@ ko +dnet:languages @=@ kur @=@ ku +dnet:languages @=@ lao @=@ lo +dnet:languages @=@ lat @=@ la +dnet:languages @=@ lav @=@ lv +dnet:languages @=@ lin @=@ ln +dnet:languages @=@ lit @=@ lt +dnet:languages @=@ mac/mak @=@ mk +dnet:languages @=@ mlg @=@ mg +dnet:languages @=@ may/msa @=@ ms +dnet:languages @=@ mlt @=@ ml +dnet:languages @=@ mao/mri @=@ mi +dnet:languages @=@ mar @=@ mr +dnet:languages @=@ mol @=@ mo +dnet:languages @=@ mon @=@ mn +dnet:languages @=@ nau @=@ na +dnet:languages @=@ nep @=@ ne +dnet:languages @=@ nor @=@ no +dnet:languages @=@ oci @=@ oc +dnet:languages @=@ ori @=@ or +dnet:languages @=@ orm @=@ om +dnet:languages @=@ pan @=@ pa +dnet:languages @=@ fas/per @=@ fa +dnet:languages @=@ pol @=@ pl +dnet:languages @=@ por @=@ pt +dnet:languages @=@ por @=@ pt_pt +dnet:languages @=@ pus @=@ ps +dnet:languages @=@ que @=@ qu +dnet:languages @=@ roh @=@ rm +dnet:languages @=@ ron/rum @=@ ro +dnet:languages @=@ run @=@ rn +dnet:languages @=@ rus @=@ ru +dnet:languages @=@ smo @=@ sm +dnet:languages @=@ sag @=@ sg +dnet:languages @=@ san @=@ sa +dnet:languages @=@ srp @=@ scc/srp +dnet:languages @=@ srp @=@ sr +dnet:languages @=@ scr @=@ sh +dnet:languages @=@ sna @=@ sn +dnet:languages @=@ snd @=@ sd +dnet:languages @=@ sin @=@ si +dnet:languages @=@ sit @=@ - +dnet:languages @=@ slk/slo @=@ sk +dnet:languages @=@ slv @=@ sl +dnet:languages @=@ som @=@ so +dnet:languages @=@ sot @=@ st +dnet:languages @=@ esl/spa @=@ es +dnet:languages @=@ sun @=@ su +dnet:languages @=@ swa @=@ sw +dnet:languages @=@ ssw @=@ ss +dnet:languages @=@ swe @=@ sv +dnet:languages @=@ swe @=@ sve/swe +dnet:languages @=@ tgl @=@ tl +dnet:languages @=@ tgk @=@ tg +dnet:languages @=@ tam @=@ ta +dnet:languages @=@ tat @=@ tt +dnet:languages @=@ tel @=@ te +dnet:languages @=@ tha @=@ th +dnet:languages @=@ bod/tib @=@ bo +dnet:languages @=@ tir @=@ ti +dnet:languages @=@ tog @=@ to +dnet:languages @=@ tso @=@ ts +dnet:languages @=@ tsn @=@ tn +dnet:languages @=@ tur @=@ tr +dnet:languages @=@ tuk @=@ tk +dnet:languages @=@ twi @=@ tw +dnet:languages @=@ uig @=@ ug +dnet:languages @=@ ukr @=@ uk +dnet:languages @=@ urd @=@ ur +dnet:languages @=@ uzb @=@ uz +dnet:languages @=@ vie @=@ vi +dnet:languages @=@ vol @=@ vo +dnet:languages @=@ wln @=@ wa +dnet:languages @=@ cym/wel @=@ cy +dnet:languages @=@ wol @=@ wo +dnet:languages @=@ xho @=@ xh +dnet:languages @=@ yid @=@ yi +dnet:languages @=@ yor @=@ yo +dnet:languages @=@ zha @=@ za +dnet:languages @=@ zul @=@ zu +dnet:pid_types @=@ orcid @=@ ORCID12 +dnet:subject_classification_typologies @=@ jel @=@ jelElement +dnet:protocols @=@ oai @=@ OAI-PMH +dnet:protocols @=@ oai @=@ OAI_PMH +dnet:publication_resource @=@ 0018 @=@ Comentario +dnet:publication_resource @=@ 0018 @=@ Comment/debate +dnet:publication_resource @=@ 0018 @=@ http://purl.org/coar/resource_type/c_1162 +dnet:publication_resource @=@ 0018 @=@ info:eu-repo/semantics/annotation +dnet:publication_resource @=@ 0001 @=@ A1 Alkuperäisartikkeli tieteellisessä aikakauslehdessä +dnet:publication_resource @=@ 0001 @=@ Article +dnet:publication_resource @=@ 0001 @=@ Article (author) +dnet:publication_resource @=@ 0001 @=@ Article - letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article / Letter to editor +dnet:publication_resource @=@ 0001 @=@ Article / Letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article / Newspaper +dnet:publication_resource @=@ 0001 @=@ Article in journal +dnet:publication_resource @=@ 0001 @=@ Article in monograph or in proceedings +dnet:publication_resource @=@ 0001 @=@ Article in proceedings +dnet:publication_resource @=@ 0001 @=@ Article-letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article/Letter to editor +dnet:publication_resource @=@ 0001 @=@ Articles in Journals +dnet:publication_resource @=@ 0001 @=@ Articolo +dnet:publication_resource @=@ 0001 @=@ Articulo +dnet:publication_resource @=@ 0001 @=@ Artículo +dnet:publication_resource @=@ 0001 @=@ Artículo - Article +dnet:publication_resource @=@ 0001 @=@ Aufsatz +dnet:publication_resource @=@ 0001 @=@ Clinical Study +dnet:publication_resource @=@ 0001 @=@ Institutional Series +dnet:publication_resource @=@ 0001 @=@ International Journal +dnet:publication_resource @=@ 0001 @=@ International Journal Abstract +dnet:publication_resource @=@ 0001 @=@ International Journal ISI/JCR +dnet:publication_resource @=@ 0001 @=@ Journal (full / special issue) +dnet:publication_resource @=@ 0001 @=@ Journal Article/Review +dnet:publication_resource @=@ 0001 @=@ Journal article +dnet:publication_resource @=@ 0001 @=@ Journal article (on-line or printed) +dnet:publication_resource @=@ 0001 @=@ Journal articles +dnet:publication_resource @=@ 0001 @=@ Journal paper +dnet:publication_resource @=@ 0001 @=@ Makale +dnet:publication_resource @=@ 0001 @=@ National Journal +dnet:publication_resource @=@ 0001 @=@ Original article (non peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Original article (peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Peer-reviewed Article +dnet:publication_resource @=@ 0001 @=@ Publication - Article +dnet:publication_resource @=@ 0001 @=@ Published Journal Article +dnet:publication_resource @=@ 0001 @=@ Research Article +dnet:publication_resource @=@ 0001 @=@ Review article (non peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Review article (peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Straipsnis / Article +dnet:publication_resource @=@ 0001 @=@ Volumes Edited / Special Issues +dnet:publication_resource @=@ 0001 @=@ art +dnet:publication_resource @=@ 0001 @=@ article in non peer-reviewed journal +dnet:publication_resource @=@ 0001 @=@ article in peer-reviewed journal +dnet:publication_resource @=@ 0001 @=@ article-commentary +dnet:publication_resource @=@ 0001 @=@ article_site_web +dnet:publication_resource @=@ 0001 @=@ artykuł +dnet:publication_resource @=@ 0001 @=@ artículo de revisión +dnet:publication_resource @=@ 0001 @=@ doc-type:Journal Article +dnet:publication_resource @=@ 0001 @=@ doc-type:article +dnet:publication_resource @=@ 0001 @=@ foly +dnet:publication_resource @=@ 0001 @=@ folyóiratcikk +dnet:publication_resource @=@ 0001 @=@ folyoiratcikk +dnet:publication_resource @=@ 0001 @=@ folyóiratcikk +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_545b +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_6501 +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_7877 +dnet:publication_resource @=@ 0001 @=@ http://purl.org/escidoc/metadata/ves/publication-types/article +dnet:publication_resource @=@ 0001 @=@ in-brief +dnet:publication_resource @=@ 0001 @=@ info:eu-repo/semantics/Journal Article +dnet:publication_resource @=@ 0001 @=@ info:eu-repo/semantics/article +dnet:publication_resource @=@ 0001 @=@ journal-article +dnet:publication_resource @=@ 0001 @=@ journalArticle +dnet:publication_resource @=@ 0001 @=@ journal_article +dnet:publication_resource @=@ 0001 @=@ letter +dnet:publication_resource @=@ 0001 @=@ non peer-reviewed article +dnet:publication_resource @=@ 0001 @=@ partial-retraction +dnet:publication_resource @=@ 0001 @=@ proceeding with peer review +dnet:publication_resource @=@ 0001 @=@ rapid-communication +dnet:publication_resource @=@ 0001 @=@ reply +dnet:publication_resource @=@ 0001 @=@ research-article +dnet:publication_resource @=@ 0001 @=@ retraction +dnet:publication_resource @=@ 0001 @=@ review-article +dnet:publication_resource @=@ 0001 @=@ text (article) +dnet:publication_resource @=@ 0001 @=@ Рецензована стаття +dnet:publication_resource @=@ 0001 @=@ Статья +dnet:publication_resource @=@ 0033 @=@ AUDIOVISUAL_DOCUMENT +dnet:publication_resource @=@ 0033 @=@ Audiovisual/Audiovisual +dnet:publication_resource @=@ 0033 @=@ http://purl.org/coar/resource_type/c_c513 +dnet:publication_resource @=@ 0008 @=@ Bachelor's +dnet:publication_resource @=@ 0008 @=@ Bachelor's Degree +dnet:publication_resource @=@ 0008 @=@ Bachelors Thesis +dnet:publication_resource @=@ 0008 @=@ Proyecto fin de carrera +dnet:publication_resource @=@ 0008 @=@ Undergraduate Thesis +dnet:publication_resource @=@ 0008 @=@ http://purl.org/coar/resource_type/c_7a1f +dnet:publication_resource @=@ 0008 @=@ info:eu-repo/semantics/bachelorThesis +dnet:publication_resource @=@ 0008 @=@ выпускная бакалаврская работа +dnet:publication_resource @=@ 0002 @=@ Book (monograph) +dnet:publication_resource @=@ 0002 @=@ Book (non peer-reviewed) +dnet:publication_resource @=@ 0002 @=@ Book (peer-reviewed) +dnet:publication_resource @=@ 0002 @=@ Book - monograph - editorial book +dnet:publication_resource @=@ 0002 @=@ Book Section +dnet:publication_resource @=@ 0002 @=@ Book as author +dnet:publication_resource @=@ 0002 @=@ Books +dnet:publication_resource @=@ 0002 @=@ Buch +dnet:publication_resource @=@ 0002 @=@ Capítulo de Libro - Book Section +dnet:publication_resource @=@ 0002 @=@ Edited Book +dnet:publication_resource @=@ 0002 @=@ International Book/Monograph +dnet:publication_resource @=@ 0002 @=@ Knyga / Book +dnet:publication_resource @=@ 0002 @=@ Książka +dnet:publication_resource @=@ 0002 @=@ Libro +dnet:publication_resource @=@ 0002 @=@ Libro - Book +dnet:publication_resource @=@ 0002 @=@ Monografia +dnet:publication_resource @=@ 0002 @=@ Monograph +dnet:publication_resource @=@ 0002 @=@ National Book/Monograph +dnet:publication_resource @=@ 0002 @=@ atlas +dnet:publication_resource @=@ 0002 @=@ book +dnet:publication_resource @=@ 0002 @=@ book-series +dnet:publication_resource @=@ 0002 @=@ book-set +dnet:publication_resource @=@ 0002 @=@ book-track +dnet:publication_resource @=@ 0002 @=@ book_series +dnet:publication_resource @=@ 0002 @=@ book_title +dnet:publication_resource @=@ 0002 @=@ doc-type:book +dnet:publication_resource @=@ 0002 @=@ eBook +dnet:publication_resource @=@ 0002 @=@ edited-book +dnet:publication_resource @=@ 0002 @=@ http://purl.org/coar/resource_type/c_2f33 +dnet:publication_resource @=@ 0002 @=@ http://purl.org/eprint/type/Book +dnet:publication_resource @=@ 0002 @=@ info:eu-repo/semantics/book +dnet:publication_resource @=@ 0002 @=@ könyv +dnet:publication_resource @=@ 0002 @=@ reference-book +dnet:publication_resource @=@ 0002 @=@ scientific book +dnet:publication_resource @=@ 0002 @=@ Книга +dnet:publication_resource @=@ 0002 @=@ Монография +dnet:publication_resource @=@ 0002 @=@ Учебник +dnet:publication_resource @=@ 0037 @=@ clinicalTrial +dnet:publication_resource @=@ 0037 @=@ http://purl.org/coar/resource_type/c_cb28 +dnet:publication_resource @=@ 0022 @=@ collection +dnet:publication_resource @=@ 0004 @=@ A4 Artikkeli konferenssijulkaisussa +dnet:publication_resource @=@ 0004 @=@ Article from Conference in a Journal +dnet:publication_resource @=@ 0004 @=@ Comunicación Congreso +dnet:publication_resource @=@ 0004 @=@ Comunicación de congreso +dnet:publication_resource @=@ 0004 @=@ Conference Abstract +dnet:publication_resource @=@ 0004 @=@ Conference Paper +dnet:publication_resource @=@ 0004 @=@ Conference Poster +dnet:publication_resource @=@ 0004 @=@ Conference Proceedings +dnet:publication_resource @=@ 0004 @=@ Conference Program +dnet:publication_resource @=@ 0004 @=@ Conference article +dnet:publication_resource @=@ 0004 @=@ Conference contribution +dnet:publication_resource @=@ 0004 @=@ Conference lecture +dnet:publication_resource @=@ 0004 @=@ Conference or Workshop Item +dnet:publication_resource @=@ 0004 @=@ Conference paper, poster, etc. +dnet:publication_resource @=@ 0004 @=@ Conference paper/abstract +dnet:publication_resource @=@ 0004 @=@ Conference papers +dnet:publication_resource @=@ 0004 @=@ Conference preprint +dnet:publication_resource @=@ 0004 @=@ Conference report +dnet:publication_resource @=@ 0004 @=@ Contributions to Conferences +dnet:publication_resource @=@ 0004 @=@ International Conference +dnet:publication_resource @=@ 0004 @=@ International Conference Abstract/Poster +dnet:publication_resource @=@ 0004 @=@ International Conference ISI/JCR +dnet:publication_resource @=@ 0004 @=@ International Conference communication/abstract/poster +dnet:publication_resource @=@ 0004 @=@ National Conference +dnet:publication_resource @=@ 0004 @=@ National Conference Abstract/Poster +dnet:publication_resource @=@ 0004 @=@ National Conference communication/abstract/poster +dnet:publication_resource @=@ 0004 @=@ PREFACE_PROCEEDINGS +dnet:publication_resource @=@ 0004 @=@ PROCEEDING_PAPER +dnet:publication_resource @=@ 0004 @=@ Papers in Conference Proceedings +dnet:publication_resource @=@ 0004 @=@ Ponencia - Conference or Workshop Item +dnet:publication_resource @=@ 0004 @=@ Presentación +dnet:publication_resource @=@ 0004 @=@ Presentation +dnet:publication_resource @=@ 0004 @=@ Proceedings (peer-reviewed) +dnet:publication_resource @=@ 0004 @=@ Proceedings of a Conference +dnet:publication_resource @=@ 0004 @=@ Proceedings paper +dnet:publication_resource @=@ 0004 @=@ Póster +dnet:publication_resource @=@ 0004 @=@ Póster de congreso +dnet:publication_resource @=@ 0004 @=@ Resumen comunicación Congreso +dnet:publication_resource @=@ 0004 @=@ Resúmen comunicación Congreso +dnet:publication_resource @=@ 0004 @=@ actas de congreso +dnet:publication_resource @=@ 0004 @=@ actes_congres +dnet:publication_resource @=@ 0004 @=@ communication_avec_actes +dnet:publication_resource @=@ 0004 @=@ communication_invitee +dnet:publication_resource @=@ 0004 @=@ communication_par_affiche +dnet:publication_resource @=@ 0004 @=@ communication_sans_actes +dnet:publication_resource @=@ 0004 @=@ conference +dnet:publication_resource @=@ 0004 @=@ conference item +dnet:publication_resource @=@ 0004 @=@ conference proceeding +dnet:publication_resource @=@ 0004 @=@ conferenceObject +dnet:publication_resource @=@ 0004 @=@ conference_paper +dnet:publication_resource @=@ 0004 @=@ doc-type:conferenceObject +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_18co +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_18cp +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_5794 +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_6670 +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_c94f +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_f744 +dnet:publication_resource @=@ 0004 @=@ http://purl.org/eprint/type/ConferencePaper +dnet:publication_resource @=@ 0004 @=@ http://purl.org/eprint/type/ConferencePoster +dnet:publication_resource @=@ 0004 @=@ http://purl.org/escidoc/metadata/ves/publication-types/conference-report +dnet:publication_resource @=@ 0004 @=@ http://purl.org/escidoc/metadata/ves/publication-types/proceedings +dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/Conference Paper +dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferenceItem +dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferenceObject +dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferencePaper +dnet:publication_resource @=@ 0004 @=@ invited conference talk +dnet:publication_resource @=@ 0004 @=@ plakat +dnet:publication_resource @=@ 0004 @=@ ponencia +dnet:publication_resource @=@ 0004 @=@ poster +dnet:publication_resource @=@ 0004 @=@ prezentacja +dnet:publication_resource @=@ 0004 @=@ proceeding, seminar, workshop without peer review +dnet:publication_resource @=@ 0004 @=@ proceedings +dnet:publication_resource @=@ 0004 @=@ proceedings-article +dnet:publication_resource @=@ 0004 @=@ научный доклад +dnet:publication_resource @=@ 0005 @=@ Magazine Article +dnet:publication_resource @=@ 0005 @=@ Newspaper Article +dnet:publication_resource @=@ 0005 @=@ Newspaper or magazine article +dnet:publication_resource @=@ 0005 @=@ contributionToPeriodical +dnet:publication_resource @=@ 0005 @=@ http://purl.org/coar/resource_type/c_998f +dnet:publication_resource @=@ 0005 @=@ info:eu-repo/semantics/contributionToPeriodical +dnet:publication_resource @=@ 0005 @=@ revista divulgativa +dnet:publication_resource @=@ 0045 @=@ Data Management Plan +dnet:publication_resource @=@ 0045 @=@ Data Management Plan (NSF Generic) +dnet:publication_resource @=@ 0045 @=@ http://purl.org/coar/resource_type/c_ab20 +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataManagementPolicy +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataManagementPolicyDocument +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataMangementPlan +dnet:publication_resource @=@ 0045 @=@ plan de gestión de datos +dnet:publication_resource @=@ 0045 @=@ publication-datamanagementplan +dnet:publication_resource @=@ 0031 @=@ Data Descriptor +dnet:publication_resource @=@ 0031 @=@ DataPaper +dnet:publication_resource @=@ 0031 @=@ data-article +dnet:publication_resource @=@ 0031 @=@ http://purl.org/coar/resource_type/c_beb9 +dnet:publication_resource @=@ 0021 @=@ Chemical Structures +dnet:publication_resource @=@ 0021 @=@ Dataset/Dataset +dnet:publication_resource @=@ 0021 @=@ Research Data +dnet:publication_resource @=@ 0021 @=@ dataset +dnet:publication_resource @=@ 0021 @=@ http://purl.org/coar/resource_type/c_ddb1 +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/DDIInstance +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/datafile +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/dataset +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/enhancedObjectFile +dnet:publication_resource @=@ 0006 @=@ Daktaro disertacija / Doctoral dissertation +dnet:publication_resource @=@ 0006 @=@ Diss +dnet:publication_resource @=@ 0006 @=@ Dissertation +dnet:publication_resource @=@ 0006 @=@ Doctoral +dnet:publication_resource @=@ 0006 @=@ DoctoralThesis +dnet:publication_resource @=@ 0006 @=@ PhD Theses +dnet:publication_resource @=@ 0006 @=@ PhD thesis +dnet:publication_resource @=@ 0006 @=@ Tenure-Promotion +dnet:publication_resource @=@ 0006 @=@ Tesi di dottorato +dnet:publication_resource @=@ 0006 @=@ Tesis +dnet:publication_resource @=@ 0006 @=@ Text.Thesis.Doctoral +dnet:publication_resource @=@ 0006 @=@ Theses +dnet:publication_resource @=@ 0006 @=@ Thesis +dnet:publication_resource @=@ 0006 @=@ Thesis or Dissertation +dnet:publication_resource @=@ 0006 @=@ Thesis.Doctoral +dnet:publication_resource @=@ 0006 @=@ doc-type:doctoralThesis +dnet:publication_resource @=@ 0006 @=@ dok +dnet:publication_resource @=@ 0006 @=@ doktori dolgozat +dnet:publication_resource @=@ 0006 @=@ http://purl.org/coar/resource_type/c_db06 +dnet:publication_resource @=@ 0006 @=@ http://purl.org/eprint/type/Thesis +dnet:publication_resource @=@ 0006 @=@ info:eu-repo/semantics/doctoralThesis +dnet:publication_resource @=@ 0006 @=@ tesis doctoral +dnet:publication_resource @=@ 0006 @=@ these +dnet:publication_resource @=@ 0006 @=@ these exercice +dnet:publication_resource @=@ 0023 @=@ Event/Event +dnet:publication_resource @=@ 0023 @=@ event +dnet:publication_resource @=@ 0009 @=@ Departmental Technical Report +dnet:publication_resource @=@ 0009 @=@ Informe Técnico +dnet:publication_resource @=@ 0009 @=@ RESEARCH_REPORT +dnet:publication_resource @=@ 0009 @=@ Tech-Report +dnet:publication_resource @=@ 0009 @=@ Technical Report +dnet:publication_resource @=@ 0009 @=@ http://purl.org/coar/resource_type/c_18gh +dnet:publication_resource @=@ 0009 @=@ informe a organismo financiador +dnet:publication_resource @=@ 0009 @=@ research report +dnet:publication_resource @=@ 0024 @=@ Video +dnet:publication_resource @=@ 0024 @=@ film +dnet:publication_resource @=@ 0024 @=@ http://purl.org/coar/resource_type/c_12ce +dnet:publication_resource @=@ 0024 @=@ http://purl.org/coar/resource_type/c_8a7e +dnet:publication_resource @=@ 0024 @=@ vídeo +dnet:publication_resource @=@ 0025 @=@ Image/Image +dnet:publication_resource @=@ 0025 @=@ Imagen +dnet:publication_resource @=@ 0025 @=@ Imagen - Image +dnet:publication_resource @=@ 0025 @=@ Imagen 3-D +dnet:publication_resource @=@ 0025 @=@ fotó +dnet:publication_resource @=@ 0025 @=@ grafika +dnet:publication_resource @=@ 0025 @=@ http://purl.org/coar/resource_type/c_ecc8 +dnet:publication_resource @=@ 0025 @=@ image +dnet:publication_resource @=@ 0025 @=@ image-diagram +dnet:publication_resource @=@ 0025 @=@ image-drawing +dnet:publication_resource @=@ 0025 @=@ image-figure +dnet:publication_resource @=@ 0025 @=@ image-other +dnet:publication_resource @=@ 0025 @=@ image-photo +dnet:publication_resource @=@ 0025 @=@ image-plot +dnet:publication_resource @=@ 0025 @=@ Drawing +dnet:publication_resource @=@ 0025 @=@ Photo +dnet:publication_resource @=@ 0025 @=@ Figure +dnet:publication_resource @=@ 0025 @=@ Diagram +dnet:publication_resource @=@ 0025 @=@ Plot +dnet:publication_resource @=@ 0026 @=@ http://purl.org/coar/resource_type/c_e9a0 +dnet:publication_resource @=@ 0026 @=@ interactiveResource +dnet:publication_resource @=@ 0011 @=@ Internal note +dnet:publication_resource @=@ 0011 @=@ http://purl.org/coar/resource_type/c_18ww +dnet:publication_resource @=@ 0043 @=@ http://purl.org/coar/resource_type/c_0640 +dnet:publication_resource @=@ 0010 @=@ Inaugural lecture +dnet:publication_resource @=@ 0010 @=@ Material didáctico +dnet:publication_resource @=@ 0010 @=@ Public-Lecture +dnet:publication_resource @=@ 0010 @=@ Teaching Resource +dnet:publication_resource @=@ 0010 @=@ http://purl.org/coar/resource_type/c_8544 +dnet:publication_resource @=@ 0010 @=@ info:eu-repo/semantics/lecture +dnet:publication_resource @=@ 0010 @=@ lesson +dnet:publication_resource @=@ 0010 @=@ Учебный материал +dnet:publication_resource @=@ 0007 @=@ Diploma Project +dnet:publication_resource @=@ 0007 @=@ MSc Thesis +dnet:publication_resource @=@ 0007 @=@ Magistro darbas / Master thesis +dnet:publication_resource @=@ 0007 @=@ Master Degree +dnet:publication_resource @=@ 0007 @=@ Master's +dnet:publication_resource @=@ 0007 @=@ Master's Degree +dnet:publication_resource @=@ 0007 @=@ Masterarbeit u.a. +dnet:publication_resource @=@ 0007 @=@ Masters (Taught) +dnet:publication_resource @=@ 0007 @=@ Masters thesis +dnet:publication_resource @=@ 0007 @=@ Masters-Thesis.Magister +dnet:publication_resource @=@ 0007 @=@ Tesina +dnet:publication_resource @=@ 0007 @=@ Thesis.Master +dnet:publication_resource @=@ 0007 @=@ Trabajo fin de Máster +dnet:publication_resource @=@ 0007 @=@ doc-type:masterThesis +dnet:publication_resource @=@ 0007 @=@ hdr +dnet:publication_resource @=@ 0007 @=@ http://purl.org/coar/resource_type/c_bdcc +dnet:publication_resource @=@ 0007 @=@ info:eu-repo/semantics/masterThesis +dnet:publication_resource @=@ 0007 @=@ masterThesis +dnet:publication_resource @=@ 0007 @=@ memoire +dnet:publication_resource @=@ 0007 @=@ tesis de maestría +dnet:publication_resource @=@ 0027 @=@ Model/Model +dnet:publication_resource @=@ 0027 @=@ model +dnet:publication_resource @=@ 0012 @=@ Newsletter Article +dnet:publication_resource @=@ 0020 @=@ Estudio y edición crítica de música +dnet:publication_resource @=@ 0020 @=@ Exhibition +dnet:publication_resource @=@ 0020 @=@ Kita / Other +dnet:publication_resource @=@ 0020 @=@ Learning Object +dnet:publication_resource @=@ 0020 @=@ Mapa +dnet:publication_resource @=@ 0020 @=@ Modelo de utilidad +dnet:publication_resource @=@ 0020 @=@ Online Resource +dnet:publication_resource @=@ 0020 @=@ Otro - Other +dnet:publication_resource @=@ 0020 @=@ PEDAGOGICAL_DOCUMENT +dnet:publication_resource @=@ 0020 @=@ Partitura +dnet:publication_resource @=@ 0020 @=@ Planimetría +dnet:publication_resource @=@ 0020 @=@ Registered Copyright +dnet:publication_resource @=@ 0020 @=@ Research Tool +dnet:publication_resource @=@ 0020 @=@ Sitio web +dnet:publication_resource @=@ 0020 @=@ Trabajo de divulgación +dnet:publication_resource @=@ 0020 @=@ University Academic Unit +dnet:publication_resource @=@ 0020 @=@ Web publication/site +dnet:publication_resource @=@ 0020 @=@ application +dnet:publication_resource @=@ 0020 @=@ artefact +dnet:publication_resource @=@ 0020 @=@ carte +dnet:publication_resource @=@ 0020 @=@ composition +dnet:publication_resource @=@ 0020 @=@ document_audiovisuel +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_12cc +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_12cd +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_1843 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_18cd +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_18cw +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_26e4 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_7ad9 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_e059 +dnet:publication_resource @=@ 0020 @=@ info:eu-repo/semantics/other +dnet:publication_resource @=@ 0020 @=@ learningObject +dnet:publication_resource @=@ 0020 @=@ map +dnet:publication_resource @=@ 0020 @=@ misc +dnet:publication_resource @=@ 0020 @=@ other +dnet:publication_resource @=@ 0020 @=@ otro +dnet:publication_resource @=@ 0020 @=@ revue +dnet:publication_resource @=@ 0038 @=@ Abstract +dnet:publication_resource @=@ 0038 @=@ Blog +dnet:publication_resource @=@ 0038 @=@ Book Prospectus +dnet:publication_resource @=@ 0038 @=@ Dictionary Entry +dnet:publication_resource @=@ 0038 @=@ Disclosure +dnet:publication_resource @=@ 0038 @=@ Editorial +dnet:publication_resource @=@ 0038 @=@ Editorial ISI/JCR +dnet:publication_resource @=@ 0038 @=@ Editors +dnet:publication_resource @=@ 0038 @=@ Editors (non peer-reviewed) +dnet:publication_resource @=@ 0038 @=@ Encyclopedia Entry +dnet:publication_resource @=@ 0038 @=@ Entrada de blog +dnet:publication_resource @=@ 0038 @=@ Funding Submission +dnet:publication_resource @=@ 0038 @=@ HabilitationThesis +dnet:publication_resource @=@ 0038 @=@ License +dnet:publication_resource @=@ 0038 @=@ Manual +dnet:publication_resource @=@ 0038 @=@ Manuscript +dnet:publication_resource @=@ 0038 @=@ Manuscrito +dnet:publication_resource @=@ 0038 @=@ Other publication (non peer-review) +dnet:publication_resource @=@ 0038 @=@ Other publication (peer-review) +dnet:publication_resource @=@ 0038 @=@ Revista +dnet:publication_resource @=@ 0038 @=@ Supervised Student Publication +dnet:publication_resource @=@ 0038 @=@ Tesis/trabajos de grado – Thesis +dnet:publication_resource @=@ 0038 @=@ Text +dnet:publication_resource @=@ 0038 @=@ Text/Text +dnet:publication_resource @=@ 0038 @=@ Trademark +dnet:publication_resource @=@ 0038 @=@ afterword +dnet:publication_resource @=@ 0038 @=@ avantpropos +dnet:publication_resource @=@ 0038 @=@ bibliography +dnet:publication_resource @=@ 0038 @=@ chronique +dnet:publication_resource @=@ 0038 @=@ compte rendu +dnet:publication_resource @=@ 0038 @=@ correction +dnet:publication_resource @=@ 0038 @=@ corrigenda +dnet:publication_resource @=@ 0038 @=@ foreword +dnet:publication_resource @=@ 0038 @=@ habilitation à diriger des recherches +dnet:publication_resource @=@ 0038 @=@ historicalDocument +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_0040 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_0857 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_18cf +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_18wz +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_3e5a +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_46ec +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_6947 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_7acd +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_86bc +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_b239 +dnet:publication_resource @=@ 0038 @=@ note de lecture +dnet:publication_resource @=@ 0038 @=@ notedelecture +dnet:publication_resource @=@ 0038 @=@ other publication +dnet:publication_resource @=@ 0038 @=@ postface +dnet:publication_resource @=@ 0038 @=@ publication-other +dnet:publication_resource @=@ 0038 @=@ revuedepresse +dnet:publication_resource @=@ 0038 @=@ sa_component +dnet:publication_resource @=@ 0038 @=@ standard +dnet:publication_resource @=@ 0038 @=@ standard-series +dnet:publication_resource @=@ 0038 @=@ Índice +dnet:publication_resource @=@ 0013 @=@ A3 Kirjan tai muun kokoomateoksen osa +dnet:publication_resource @=@ 0013 @=@ Book Part (author) +dnet:publication_resource @=@ 0013 @=@ Book Section / Chapter +dnet:publication_resource @=@ 0013 @=@ Book chapter or Essay in book +dnet:publication_resource @=@ 0013 @=@ Book editorial +dnet:publication_resource @=@ 0013 @=@ Book section +dnet:publication_resource @=@ 0013 @=@ Book_Chapter +dnet:publication_resource @=@ 0013 @=@ Buchbeitrag +dnet:publication_resource @=@ 0013 @=@ Capítulo de libro +dnet:publication_resource @=@ 0013 @=@ Contribution to International Book/Monograph +dnet:publication_resource @=@ 0013 @=@ Contribution to International Book/Monograph ISI/JCR +dnet:publication_resource @=@ 0013 @=@ Contribution to National Book/Monograph +dnet:publication_resource @=@ 0013 @=@ Contribution to book (non peer-reviewed) +dnet:publication_resource @=@ 0013 @=@ Contribution to book (peer-reviewed) +dnet:publication_resource @=@ 0013 @=@ Knygos dalis / Book chapter +dnet:publication_resource @=@ 0013 @=@ Part of book - chapter +dnet:publication_resource @=@ 0013 @=@ Rozdział z książki +dnet:publication_resource @=@ 0013 @=@ book chapter +dnet:publication_resource @=@ 0013 @=@ book-chapter +dnet:publication_resource @=@ 0013 @=@ book-part +dnet:publication_resource @=@ 0013 @=@ book-section +dnet:publication_resource @=@ 0013 @=@ bookPart +dnet:publication_resource @=@ 0013 @=@ book_content +dnet:publication_resource @=@ 0013 @=@ chapitre_ouvrage +dnet:publication_resource @=@ 0013 @=@ chapter +dnet:publication_resource @=@ 0013 @=@ doc-type:bookPart +dnet:publication_resource @=@ 0013 @=@ http://purl.org/coar/resource_type/c_3248 +dnet:publication_resource @=@ 0013 @=@ http://purl.org/eprint/type/BookItem +dnet:publication_resource @=@ 0013 @=@ info:eu-repo/semantics/Chapter +dnet:publication_resource @=@ 0013 @=@ info:eu-repo/semantics/bookPart +dnet:publication_resource @=@ 0013 @=@ könyvfejezet +dnet:publication_resource @=@ 0013 @=@ publication-section +dnet:publication_resource @=@ 0013 @=@ reference-entry +dnet:publication_resource @=@ 0013 @=@ reference_entry +dnet:publication_resource @=@ 0013 @=@ scientific book chapter +dnet:publication_resource @=@ 0013 @=@ Глава монографии +dnet:publication_resource @=@ 0019 @=@ H1 Myönnetty patentti +dnet:publication_resource @=@ 0019 @=@ Patente +dnet:publication_resource @=@ 0019 @=@ Solicitud de patente +dnet:publication_resource @=@ 0019 @=@ Traducción de patente +dnet:publication_resource @=@ 0019 @=@ brevet +dnet:publication_resource @=@ 0019 @=@ http://purl.org/coar/resource_type/c_15cd +dnet:publication_resource @=@ 0019 @=@ info:eu-repo/semantics/patent +dnet:publication_resource @=@ 0028 @=@ Service +dnet:publication_resource @=@ 0028 @=@ physicalObject +dnet:publication_resource @=@ 0016 @=@ Pre Print +dnet:publication_resource @=@ 0016 @=@ Pre-print +dnet:publication_resource @=@ 0016 @=@ http://purl.org/coar/resource_type/c_816b +dnet:publication_resource @=@ 0016 @=@ info:eu-repo/semantics/preprint +dnet:publication_resource @=@ 0016 @=@ Препринт +dnet:publication_resource @=@ 0034 @=@ http://purl.org/coar/resource_type/c_18op +dnet:publication_resource @=@ 0034 @=@ publication-deliverable +dnet:publication_resource @=@ 0034 @=@ Project deliverable +dnet:publication_resource @=@ 0035 @=@ publication-milestone +dnet:publication_resource @=@ 0035 @=@ Project milestone +dnet:publication_resource @=@ 0036 @=@ http://purl.org/coar/resource_type/c_baaf +dnet:publication_resource @=@ 0036 @=@ research-proposal +dnet:publication_resource @=@ 0036 @=@ Proposal +dnet:publication_resource @=@ 0017 @=@ ACTIVITY_REPORT +dnet:publication_resource @=@ 0017 @=@ Case Report +dnet:publication_resource @=@ 0017 @=@ Commissioned report +dnet:publication_resource @=@ 0017 @=@ D4 Julkaistu kehittämis- tai tutkimusraportti tai -selvitys +dnet:publication_resource @=@ 0017 @=@ Deliverable +dnet:publication_resource @=@ 0017 @=@ Documento tecnico +dnet:publication_resource @=@ 0017 @=@ EUR - Scientific and Technical Research Reports +dnet:publication_resource @=@ 0017 @=@ JRC Reference Reports +dnet:publication_resource @=@ 0017 @=@ Project Report +dnet:publication_resource @=@ 0017 @=@ brief-report +dnet:publication_resource @=@ 0017 @=@ case-report +dnet:publication_resource @=@ 0017 @=@ chapitre_rapport +dnet:publication_resource @=@ 0017 @=@ doc-type:report +dnet:publication_resource @=@ 0017 @=@ document_institutionnel +dnet:publication_resource @=@ 0017 @=@ document_technique +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_186u +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18hj +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18wq +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18ws +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_71bd +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_93fc +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_ba1f +dnet:publication_resource @=@ 0017 @=@ http://purl.org/eprint/type/Report +dnet:publication_resource @=@ 0017 @=@ info:eu-repo/semantics/report +dnet:publication_resource @=@ 0017 @=@ publication-softwaredocumentation +dnet:publication_resource @=@ 0017 @=@ rapport_expertise +dnet:publication_resource @=@ 0017 @=@ rapport_mission +dnet:publication_resource @=@ 0017 @=@ report +dnet:publication_resource @=@ 0017 @=@ report-paper +dnet:publication_resource @=@ 0017 @=@ report-paper_title +dnet:publication_resource @=@ 0017 @=@ report-series +dnet:publication_resource @=@ 0017 @=@ support_cours +dnet:publication_resource @=@ 0017 @=@ Software documentation +dnet:publication_resource @=@ 0014 @=@ Arbeitspapier +dnet:publication_resource @=@ 0014 @=@ Departmental Bulletin Paper +dnet:publication_resource @=@ 0014 @=@ Documento de trabajo +dnet:publication_resource @=@ 0014 @=@ Paper +dnet:publication_resource @=@ 0014 @=@ Project description +dnet:publication_resource @=@ 0014 @=@ Research-Paper +dnet:publication_resource @=@ 0014 @=@ ResearchPaper +dnet:publication_resource @=@ 0014 @=@ Working / discussion paper +dnet:publication_resource @=@ 0014 @=@ Working Paper +dnet:publication_resource @=@ 0014 @=@ Working Paper / Technical Report +dnet:publication_resource @=@ 0014 @=@ doc-type:workingPaper +dnet:publication_resource @=@ 0014 @=@ http://purl.org/coar/resource_type/c_8042 +dnet:publication_resource @=@ 0014 @=@ http://purl.org/escidoc/metadata/ves/publication-types/paper +dnet:publication_resource @=@ 0014 @=@ info:eu-repo/semantics/paper +dnet:publication_resource @=@ 0014 @=@ info:eu-repo/semantics/workingPaper +dnet:publication_resource @=@ 0014 @=@ workingPaper +dnet:publication_resource @=@ 0015 @=@ A2 Katsausartikkeli tieteellisessä aikakauslehdessä +dnet:publication_resource @=@ 0015 @=@ Book Review +dnet:publication_resource @=@ 0015 @=@ Book/Film/Article review +dnet:publication_resource @=@ 0015 @=@ Literature review +dnet:publication_resource @=@ 0015 @=@ Peer review +dnet:publication_resource @=@ 0015 @=@ Reseña bibliográfica +dnet:publication_resource @=@ 0015 @=@ Review Article +dnet:publication_resource @=@ 0015 @=@ Revisión +dnet:publication_resource @=@ 0015 @=@ RezensionReview +dnet:publication_resource @=@ 0015 @=@ book-review +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_ba08 +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_dcae04bc +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_efa0 +dnet:publication_resource @=@ 0015 @=@ info:eu-repo/semantics/review +dnet:publication_resource @=@ 0015 @=@ peer-review +dnet:publication_resource @=@ 0015 @=@ reseña de libro +dnet:publication_resource @=@ 0029 @=@ Software +dnet:publication_resource @=@ 0029 @=@ Software/Software +dnet:publication_resource @=@ 0029 @=@ Workflow +dnet:publication_resource @=@ 0029 @=@ Workflow/Workflow +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_393c +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_5ce6 +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_c950 +dnet:publication_resource @=@ 0032 @=@ http://purl.org/coar/resource_type/c_7bab +dnet:publication_resource @=@ 0030 @=@ Audio +dnet:publication_resource @=@ 0030 @=@ http://purl.org/coar/resource_type/c_18cc +dnet:publication_resource @=@ 0030 @=@ sound +dnet:publication_resource @=@ 0044 @=@ Graduate diploma +dnet:publication_resource @=@ 0044 @=@ Undergraduate diploma +dnet:review_levels @=@ 0002 @=@ 80 大阪経大学会「Working Paper」 +dnet:review_levels @=@ 0002 @=@ AO +dnet:review_levels @=@ 0002 @=@ ARTICLE SANS COMITE DE LECTURE (ASCL) +dnet:review_levels @=@ 0002 @=@ Arbeitspapier +dnet:review_levels @=@ 0002 @=@ Arbeitspapier [workingPaper] +dnet:review_levels @=@ 0002 @=@ Article (author) +dnet:review_levels @=@ 0002 @=@ Article type: preprint +dnet:review_levels @=@ 0002 @=@ Article(author version) +dnet:review_levels @=@ 0002 @=@ Article, not peer-reviewed +dnet:review_levels @=@ 0002 @=@ Articulo no evaluado +dnet:review_levels @=@ 0002 @=@ Artigo Solicitado e Não Avaliado por Pares +dnet:review_levels @=@ 0002 @=@ Artigo não avaliado pelos pares +dnet:review_levels @=@ 0002 @=@ Artigo não avaliado por pares +dnet:review_levels @=@ 0002 @=@ Artigo não avaliado por pres +dnet:review_levels @=@ 0002 @=@ Artikkeli|Artikkeli ammattilehdessä. Ei vertaisarvioitu +dnet:review_levels @=@ 0002 @=@ Artículo no evaluado +dnet:review_levels @=@ 0002 @=@ Book (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ Book Part (author) +dnet:review_levels @=@ 0002 @=@ Book item; Non-peer-reviewed +dnet:review_levels @=@ 0002 @=@ Conference preprint +dnet:review_levels @=@ 0002 @=@ Contribution to book (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ Discussion Paper +dnet:review_levels @=@ 0002 @=@ Document de travail (Working Paper) +dnet:review_levels @=@ 0002 @=@ Documento de trabajo +dnet:review_levels @=@ 0002 @=@ Documento de trabajo de investigaci??n +dnet:review_levels @=@ 0002 @=@ E-pub ahead of print +dnet:review_levels @=@ 0002 @=@ Editorial de revista, no evaluado por pares +dnet:review_levels @=@ 0002 @=@ Editorial de revista, não avaliado por pares +dnet:review_levels @=@ 0002 @=@ Editorial não avaliado pelos pares +dnet:review_levels @=@ 0002 @=@ Editors (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ Epub ahead of print +dnet:review_levels @=@ 0002 @=@ Hakemlik Sürecinden Geçmiş Makale +dnet:review_levels @=@ 0002 @=@ Hakemlik sürecindeki makale +dnet:review_levels @=@ 0002 @=@ Hakemlik sürecinden geçmemiş kitap değerlendirmesi +dnet:review_levels @=@ 0002 @=@ Journal Article (author version) +dnet:review_levels @=@ 0002 @=@ Journal Article Preprint +dnet:review_levels @=@ 0002 @=@ Journal Editorial, not peer-reviewed +dnet:review_levels @=@ 0002 @=@ Journal article; Non-peer-reviewed +dnet:review_levels @=@ 0002 @=@ Journal:WorkingPaper +dnet:review_levels @=@ 0002 @=@ Manuscript (preprint) +dnet:review_levels @=@ 0002 @=@ Monográfico (Informes, Documentos de trabajo, etc.) +dnet:review_levels @=@ 0002 @=@ NOTE INTERNE OU DE TRAVAIL +dnet:review_levels @=@ 0002 @=@ Nicht begutachteter Beitrag +dnet:review_levels @=@ 0002 @=@ No evaluado por pares +dnet:review_levels @=@ 0002 @=@ Non-Refereed +dnet:review_levels @=@ 0002 @=@ Non-refeered article +dnet:review_levels @=@ 0002 @=@ Non-refereed Article +dnet:review_levels @=@ 0002 @=@ Non-refereed Book Review +dnet:review_levels @=@ 0002 @=@ Non-refereed Review +dnet:review_levels @=@ 0002 @=@ Non-refereed Text +dnet:review_levels @=@ 0002 @=@ NonPeerReviewed +dnet:review_levels @=@ 0002 @=@ Not Peer reviewed +dnet:review_levels @=@ 0002 @=@ Not Reviewed +dnet:review_levels @=@ 0002 @=@ Not peer-reviewed +dnet:review_levels @=@ 0002 @=@ Não Avaliado por Pares +dnet:review_levels @=@ 0002 @=@ Não avaliada pelos pares +dnet:review_levels @=@ 0002 @=@ Não avaliado pelos pares +dnet:review_levels @=@ 0002 @=@ Original article (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ Other publication (non peer-review) +dnet:review_levels @=@ 0002 @=@ Pre Print +dnet:review_levels @=@ 0002 @=@ Pre-print +dnet:review_levels @=@ 0002 @=@ Preprint Article +dnet:review_levels @=@ 0002 @=@ Preprints +dnet:review_levels @=@ 0002 @=@ Preprints, Working Papers, ... +dnet:review_levels @=@ 0002 @=@ Rapporto tecnico / Working Paper / Rapporto di progetto +dnet:review_levels @=@ 0002 @=@ Resumo Não Avaliado por Pares +dnet:review_levels @=@ 0002 @=@ Review article (non peer-reviewed) +dnet:review_levels @=@ 0002 @=@ SMUR +dnet:review_levels @=@ 0002 @=@ Submissão dos artigos +dnet:review_levels @=@ 0002 @=@ Submitted version +dnet:review_levels @=@ 0002 @=@ Vertaisarvioimaton kirjan tai muun kokoomateoksen osa +dnet:review_levels @=@ 0002 @=@ Vorabdruck +dnet:review_levels @=@ 0002 @=@ Wetensch. publ. non-refereed +dnet:review_levels @=@ 0002 @=@ Working / discussion paper +dnet:review_levels @=@ 0002 @=@ Working Document +dnet:review_levels @=@ 0002 @=@ Working Notes +dnet:review_levels @=@ 0002 @=@ Working Paper +dnet:review_levels @=@ 0002 @=@ Working Paper / Technical Report +dnet:review_levels @=@ 0002 @=@ Working Papers +dnet:review_levels @=@ 0002 @=@ WorkingPaper +dnet:review_levels @=@ 0002 @=@ article in non peer-reviewed journal +dnet:review_levels @=@ 0002 @=@ articolo preliminare +dnet:review_levels @=@ 0002 @=@ articulo preliminar +dnet:review_levels @=@ 0002 @=@ articulo sin revision por pares +dnet:review_levels @=@ 0002 @=@ artigo preliminar +dnet:review_levels @=@ 0002 @=@ artigo sem revisão +dnet:review_levels @=@ 0002 @=@ artículo preliminar +dnet:review_levels @=@ 0002 @=@ artículo sin revisión por pares +dnet:review_levels @=@ 0002 @=@ bookchapter (author version) +dnet:review_levels @=@ 0002 @=@ borrador +dnet:review_levels @=@ 0002 @=@ column (author version) +dnet:review_levels @=@ 0002 @=@ communication_invitee +dnet:review_levels @=@ 0002 @=@ doc-type:preprint +dnet:review_levels @=@ 0002 @=@ doc-type:workingPaper +dnet:review_levels @=@ 0002 @=@ draf +dnet:review_levels @=@ 0002 @=@ draft +dnet:review_levels @=@ 0002 @=@ eu-repo/semantics/submittedVersion +dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/resource_type/c_8042 +dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/resource_type/c_816b +dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/version/c_71e4c1898caa6e32 +dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/version/c_b1a7d7d4d402bcce +dnet:review_levels @=@ 0002 @=@ http://purl.org/eprint/type/SubmittedBookItem +dnet:review_levels @=@ 0002 @=@ http://purl.org/eprint/type/SubmittedJournalArticle +dnet:review_levels @=@ 0002 @=@ http://purl.org/info:eu-repo/semantics/authorVersion +dnet:review_levels @=@ 0002 @=@ http://purl.org/info:eu-repo/semantics/submittedVersion +dnet:review_levels @=@ 0002 @=@ http://purl.org/spar/fabio/Preprint +dnet:review_levels @=@ 0002 @=@ http://purl.org/spar/fabio/WorkingPaper +dnet:review_levels @=@ 0002 @=@ https://dictionary.casrai.org/Preprint +dnet:review_levels @=@ 0002 @=@ info:ar-repo/semantics/documento de trabajo +dnet:review_levels @=@ 0002 @=@ info:ar-repo/semantics/documentoDeTrabajo +dnet:review_levels @=@ 0002 @=@ info:eu repo/semantics/draft +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/authorVersion +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/draft +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/preprint +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/submitedVersion +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/submittedVersion +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/unReviewed +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/updatedVersion +dnet:review_levels @=@ 0002 @=@ info:eu-repo/semantics/workingPaper +dnet:review_levels @=@ 0002 @=@ info:eu-repo/submittedVersion +dnet:review_levels @=@ 0002 @=@ info:ulb-repo/semantics/articleNonPeerReview +dnet:review_levels @=@ 0002 @=@ info:ulb-repo/semantics/openurl/vlink-workingpaper +dnet:review_levels @=@ 0002 @=@ info:ulb-repo/semantics/workingPaper +dnet:review_levels @=@ 0002 @=@ non peer-reviewed article +dnet:review_levels @=@ 0002 @=@ non-refereed review article +dnet:review_levels @=@ 0002 @=@ não avaliado +dnet:review_levels @=@ 0002 @=@ preprint +dnet:review_levels @=@ 0002 @=@ prepublicación +dnet:review_levels @=@ 0002 @=@ proceeding, seminar, workshop without peer review +dnet:review_levels @=@ 0002 @=@ proceedings (author version) +dnet:review_levels @=@ 0002 @=@ pré-print +dnet:review_levels @=@ 0002 @=@ pré-publication +dnet:review_levels @=@ 0002 @=@ préprint +dnet:review_levels @=@ 0002 @=@ prépublication +dnet:review_levels @=@ 0002 @=@ publicació preliminar +dnet:review_levels @=@ 0002 @=@ publication-preprint +dnet:review_levels @=@ 0002 @=@ publication-workingpaper +dnet:review_levels @=@ 0002 @=@ submitedVersion +dnet:review_levels @=@ 0002 @=@ submittedVersion +dnet:review_levels @=@ 0002 @=@ voordruk +dnet:review_levels @=@ 0002 @=@ workingPaper +dnet:review_levels @=@ 0002 @=@ ön baskı +dnet:review_levels @=@ 0002 @=@ Препринт +dnet:review_levels @=@ 0002 @=@ предпечатная версия публикации +dnet:review_levels @=@ 0002 @=@ препринт статьи +dnet:review_levels @=@ 0002 @=@ ディスカッション／ワーキング・ペーパー DP/WP +dnet:review_levels @=@ 0002 @=@ プレプリント +dnet:review_levels @=@ 0002 @=@ プレプリント Preprint +dnet:review_levels @=@ 0002 @=@ プレプリント（Preprint） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-その他（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-テクニカルレポート類（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-会議発表論文（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-図書（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-学術雑誌論文（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-紀要論文（査読無し） +dnet:review_levels @=@ 0002 @=@ 印刷物/電子媒体-雑誌記事（査読無し） +dnet:review_levels @=@ 0002 @=@ 预印本 +dnet:review_levels @=@ 0001 @=@ ##rt.metadata.pkp.peerReviewed## +dnet:review_levels @=@ 0001 @=@ A1 Alkuperäisartikkeli tieteellisessä aikakauslehdessä +dnet:review_levels @=@ 0001 @=@ Art?culo revisado por pares +dnet:review_levels @=@ 0001 @=@ Article revisat per persones expertes +dnet:review_levels @=@ 0001 @=@ Article type: peer review +dnet:review_levels @=@ 0001 @=@ Article évalué par les pairs +dnet:review_levels @=@ 0001 @=@ Article évalué par des pairs +dnet:review_levels @=@ 0001 @=@ Article évalué par les pairs +dnet:review_levels @=@ 0001 @=@ Articolo valutato secondo i criteri della peer review +dnet:review_levels @=@ 0001 @=@ Articulo evaluado por dos pares +dnet:review_levels @=@ 0001 @=@ Articulo revisado por pares +dnet:review_levels @=@ 0001 @=@ Artigo Avaliado pelos Pares +dnet:review_levels @=@ 0001 @=@ Artigo Revisto por Pares +dnet:review_levels @=@ 0001 @=@ Artigo avaliado por blind peer review +dnet:review_levels @=@ 0001 @=@ Artigo avaliado por pares +dnet:review_levels @=@ 0001 @=@ Artigo de convidado. Avaliado pelos pares +dnet:review_levels @=@ 0001 @=@ Artigos; Avaliado pelos pares +dnet:review_levels @=@ 0001 @=@ Artículo de investigación, Investigaciones originales, Artículo evaluado por pares, Investigaciones empíricas +dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares +dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares, Ensayos de investigación +dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares, Investigaciones empíricas, Artículos de investigación +dnet:review_levels @=@ 0001 @=@ Artículo revisado por pares +dnet:review_levels @=@ 0001 @=@ Artículos de estudiantes, Artículo evaluado por pares, Artículos de investigación +dnet:review_levels @=@ 0001 @=@ Artículos de investigación evaluados por doble ciego +dnet:review_levels @=@ 0001 @=@ Artículos evaluadores por doble ciego +dnet:review_levels @=@ 0001 @=@ Artículos evaluados por pares +dnet:review_levels @=@ 0001 @=@ Artículos evaluados por pares académicos +dnet:review_levels @=@ 0001 @=@ Artículos revisados por pares +dnet:review_levels @=@ 0001 @=@ avaliadas pelos pares +dnet:review_levels @=@ 0001 @=@ Avaliado anonimamente por pares +dnet:review_levels @=@ 0001 @=@ Avaliado em duplo cego por pares +dnet:review_levels @=@ 0001 @=@ Avaliado pela Editoria +dnet:review_levels @=@ 0001 @=@ Avaliado pela Editoria. Avaliado pelos pares. +dnet:review_levels @=@ 0001 @=@ Avaliado pelo Editoria +dnet:review_levels @=@ 0001 @=@ Avaliado pelo pares +dnet:review_levels @=@ 0001 @=@ Avaliado pelos Editores +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares, Artigo de convidado +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares, Artigos Originais +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares, Artigos Originais, Artigos de Revisão +dnet:review_levels @=@ 0001 @=@ Avaliado pelos pares. Avaliado pelo Editoria +dnet:review_levels @=@ 0001 @=@ Avaliado po Pares +dnet:review_levels @=@ 0001 @=@ Avaliado por Editor +dnet:review_levels @=@ 0001 @=@ Avaliado por pares +dnet:review_levels @=@ 0001 @=@ Avaliados pelos pares +dnet:review_levels @=@ 0001 @=@ Avaliados por Pares +dnet:review_levels @=@ 0001 @=@ Blind Peer-reviewed Article +dnet:review_levels @=@ 0001 @=@ Book (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Comentario de libros, Comentario de revistas, Comentario de conferencias, Artículo evaluado por pares, Artículo de investigación +dnet:review_levels @=@ 0001 @=@ Conference paper; Peer-reviewed +dnet:review_levels @=@ 0001 @=@ Contribution to book (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Documento Avaliado por Pares +dnet:review_levels @=@ 0001 @=@ Double blind evaluation articles +dnet:review_levels @=@ 0001 @=@ Double blind peer review +dnet:review_levels @=@ 0001 @=@ Editors (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Evaluación por pares +dnet:review_levels @=@ 0001 @=@ Evaluado por pares +dnet:review_levels @=@ 0001 @=@ Evaluados por los pares +dnet:review_levels @=@ 0001 @=@ Hakem sürecinden geçmiş makale +dnet:review_levels @=@ 0001 @=@ Hakemli makale +dnet:review_levels @=@ 0001 @=@ Hakemlik Sürecinden Geçmiş +dnet:review_levels @=@ 0001 @=@ Invited Peer-Reviewed Article +dnet:review_levels @=@ 0001 @=@ Journal article; Peer-reviewed +dnet:review_levels @=@ 0001 @=@ Original article (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Other publication (peer-review) +dnet:review_levels @=@ 0001 @=@ Paper peer-reviewed +dnet:review_levels @=@ 0001 @=@ Papers evaluated by academic peers +dnet:review_levels @=@ 0001 @=@ Peer reviewed +dnet:review_levels @=@ 0001 @=@ Peer reviewed article +dnet:review_levels @=@ 0001 @=@ Peer reviewed invited commentry +dnet:review_levels @=@ 0001 @=@ Peer-Reviewed Protocol +dnet:review_levels @=@ 0001 @=@ Peer-reviewd Article +dnet:review_levels @=@ 0001 @=@ Peer-reviewed +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Article +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Paper +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Review +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Review Article +dnet:review_levels @=@ 0001 @=@ Peer-reviewed Text +dnet:review_levels @=@ 0001 @=@ Peer-reviewed communication +dnet:review_levels @=@ 0001 @=@ Peer-reviewed research article +dnet:review_levels @=@ 0001 @=@ Peer-reviewed short communication +dnet:review_levels @=@ 0001 @=@ PeerReviewed +dnet:review_levels @=@ 0001 @=@ Proceedings (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Refereed +dnet:review_levels @=@ 0001 @=@ Refereed Article +dnet:review_levels @=@ 0001 @=@ Research articles evaluated by double blind +dnet:review_levels @=@ 0001 @=@ Resenha avaliada pelos pares +dnet:review_levels @=@ 0001 @=@ Review article (peer-reviewed) +dnet:review_levels @=@ 0001 @=@ Reviewed by peers +dnet:review_levels @=@ 0001 @=@ Revisión por Expertos +dnet:review_levels @=@ 0001 @=@ Revisto por Pares +dnet:review_levels @=@ 0001 @=@ SBBq abstracts / peer-reviewed +dnet:review_levels @=@ 0001 @=@ SBBq resúmenes - revisada por pares +dnet:review_levels @=@ 0001 @=@ Scholarly publ. Refereed +dnet:review_levels @=@ 0001 @=@ Scientific Publ (refereed) +dnet:review_levels @=@ 0001 @=@ Vertaisarvioimaton kirjoitus tieteellisessä aikakauslehdessä +dnet:review_levels @=@ 0001 @=@ Vertaisarvioitu alkuperäisartikkeli tieteellisessä aikakauslehdessä +dnet:review_levels @=@ 0001 @=@ Vertaisarvioitu artikkeli konferenssijulkaisussa +dnet:review_levels @=@ 0001 @=@ Vertaisarvioitu artikkeli tieteellisessä aikakauslehdessä +dnet:review_levels @=@ 0001 @=@ Vertaisarvioitu kirjan tai muun kokoomateoksen osa +dnet:review_levels @=@ 0001 @=@ Wetensch. publ. Refereed +dnet:review_levels @=@ 0001 @=@ article in peer-reviewed journal +dnet:review_levels @=@ 0001 @=@ articles validés +dnet:review_levels @=@ 0001 @=@ avaliado por pares, temas livres +dnet:review_levels @=@ 0001 @=@ info:eu-repo/semantics/peerReviewed +dnet:review_levels @=@ 0001 @=@ info:ulb-repo/semantics/articlePeerReview +dnet:review_levels @=@ 0001 @=@ proceeding with peer review +dnet:review_levels @=@ 0001 @=@ refereed_publications +dnet:review_levels @=@ 0001 @=@ ul_published_reviewed +dnet:review_levels @=@ 0001 @=@ Άρθρο που έχει αξιολογηθεί από ομότιμους ειδικούς +dnet:review_levels @=@ 0001 @=@ Άρθρο το οποίο έχει περάσει από ομότιμη αξιολόγηση +dnet:review_levels @=@ 0001 @=@ レフェリー付き論文 +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-テクニカルレポート類（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-会議発表論文（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-図書（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-学術雑誌論文（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-紀要論文（査読有り） +dnet:review_levels @=@ 0001 @=@ 印刷物/電子媒体-雑誌記事（査読有り） +dnet:review_levels @=@ 0001 @=@ 原著論文（査読有り） +dnet:review_levels @=@ 0001 @=@ 査読論文 \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt new file mode 100644 index 000000000..1daca3ca9 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt @@ -0,0 +1,1044 @@ +ModularUiLabels @=@ ModularUiLabels @=@ PendingRepositoryResources @=@ Pending datasource +ModularUiLabels @=@ ModularUiLabels @=@ RepositoryServiceResources @=@ Valid datasource +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/OPENACCESS_VERSION @=@ Another Open Access version of a publication +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/PID @=@ Another persistent identifier associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/MESHEUROPMC @=@ Another classification term from the Medical Subject Headings (MeSH) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/DDC @=@ Another Dewey Decimal classification term (DDC) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/ACM @=@ Another ACM classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/JEL @=@ Another Journal of Economic Literature (JEL) classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/ARXIV @=@ Another ARXIV classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PID @=@ A persistent identifier associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PROJECT @=@ A project reference that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/ABSTRACT @=@ An abstract describing among your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/MESHEUROPMC @=@ A classification term from the Medical Subject Headings (MeSH) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/DDC @=@ A Dewey Decimal classification term (DDC) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/ACM @=@ An ACM classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/JEL @=@ A Journal of Economic Literature (JEL) classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/ARXIV @=@ An ARXIV classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION_DATE @=@ A date of publication missing in your content +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/OPENACCESS_VERSION @=@ An Open Access versions of your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/AUTHOR/ORCID @=@ An Open Researcher and Contributor ID (ORCID) that can be associated to an author of your publications +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsCitedBy @=@ IsCitedBy +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsNewVersionOf @=@ IsNewVersionOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsPartOf @=@ IsPartOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsPreviousVersionOf @=@ IsPreviousVersionOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsReferencedBy @=@ IsReferencedBy +datacite:relation_typologies @=@ datacite:relation_typologies @=@ References @=@ References +datacite:relation_typologies @=@ datacite:relation_typologies @=@ UNKNOWN @=@ UNKNOWN +datacite:id_typologies @=@ datacite:id_typologies @=@ ARK @=@ ARK +datacite:id_typologies @=@ datacite:id_typologies @=@ DOI @=@ DOI +datacite:id_typologies @=@ datacite:id_typologies @=@ EAN13 @=@ EAN13 +datacite:id_typologies @=@ datacite:id_typologies @=@ EISSN @=@ EISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ Handle @=@ Handle +datacite:id_typologies @=@ datacite:id_typologies @=@ ISBN @=@ ISBN +datacite:id_typologies @=@ datacite:id_typologies @=@ ISSN @=@ ISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ ISTC @=@ ISTC +datacite:id_typologies @=@ datacite:id_typologies @=@ LISSN @=@ LISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ LSID @=@ LSID +datacite:id_typologies @=@ datacite:id_typologies @=@ PURL @=@ PURL +datacite:id_typologies @=@ datacite:id_typologies @=@ UNKNOWN @=@ UNKNOWN +datacite:id_typologies @=@ datacite:id_typologies @=@ UPC @=@ UPC +datacite:id_typologies @=@ datacite:id_typologies @=@ URL @=@ URL +datacite:id_typologies @=@ datacite:id_typologies @=@ URN @=@ URN +dnet:countries @=@ dnet:countries @=@ AF @=@ Afghanistan +dnet:countries @=@ dnet:countries @=@ AL @=@ Albania +dnet:countries @=@ dnet:countries @=@ DZ @=@ Algeria +dnet:countries @=@ dnet:countries @=@ AS @=@ American Samoa +dnet:countries @=@ dnet:countries @=@ AD @=@ Andorra +dnet:countries @=@ dnet:countries @=@ AO @=@ Angola +dnet:countries @=@ dnet:countries @=@ AI @=@ Anguilla +dnet:countries @=@ dnet:countries @=@ AQ @=@ Antarctica +dnet:countries @=@ dnet:countries @=@ AG @=@ Antigua and Barbuda +dnet:countries @=@ dnet:countries @=@ AR @=@ Argentina +dnet:countries @=@ dnet:countries @=@ AM @=@ Armenia +dnet:countries @=@ dnet:countries @=@ AW @=@ Aruba +dnet:countries @=@ dnet:countries @=@ AU @=@ Australia +dnet:countries @=@ dnet:countries @=@ AT @=@ Austria +dnet:countries @=@ dnet:countries @=@ AZ @=@ Azerbaijan +dnet:countries @=@ dnet:countries @=@ BS @=@ Bahamas +dnet:countries @=@ dnet:countries @=@ BH @=@ Bahrain +dnet:countries @=@ dnet:countries @=@ BD @=@ Bangladesh +dnet:countries @=@ dnet:countries @=@ BB @=@ Barbados +dnet:countries @=@ dnet:countries @=@ BY @=@ Belarus +dnet:countries @=@ dnet:countries @=@ BE @=@ Belgium +dnet:countries @=@ dnet:countries @=@ BZ @=@ Belize +dnet:countries @=@ dnet:countries @=@ BJ @=@ Benin +dnet:countries @=@ dnet:countries @=@ BM @=@ Bermuda +dnet:countries @=@ dnet:countries @=@ BT @=@ Bhutan +dnet:countries @=@ dnet:countries @=@ BO @=@ Bolivia +dnet:countries @=@ dnet:countries @=@ BQ @=@ Bonaire, Sint Eustatius and Saba +dnet:countries @=@ dnet:countries @=@ BA @=@ Bosnia and Herzegovina +dnet:countries @=@ dnet:countries @=@ BW @=@ Botswana +dnet:countries @=@ dnet:countries @=@ BV @=@ Bouvet Island +dnet:countries @=@ dnet:countries @=@ BR @=@ Brazil +dnet:countries @=@ dnet:countries @=@ IO @=@ British Indian Ocean Territory +dnet:countries @=@ dnet:countries @=@ BN @=@ Brunei Darussalam +dnet:countries @=@ dnet:countries @=@ BG @=@ Bulgaria +dnet:countries @=@ dnet:countries @=@ BF @=@ Burkina Faso +dnet:countries @=@ dnet:countries @=@ BI @=@ Burundi +dnet:countries @=@ dnet:countries @=@ KH @=@ Cambodia +dnet:countries @=@ dnet:countries @=@ CM @=@ Cameroon +dnet:countries @=@ dnet:countries @=@ CA @=@ Canada +dnet:countries @=@ dnet:countries @=@ CV @=@ Cape Verde +dnet:countries @=@ dnet:countries @=@ KY @=@ Cayman Islands +dnet:countries @=@ dnet:countries @=@ CF @=@ Central African Republic +dnet:countries @=@ dnet:countries @=@ TD @=@ Chad +dnet:countries @=@ dnet:countries @=@ CL @=@ Chile +dnet:countries @=@ dnet:countries @=@ CN @=@ China (People's Republic of) +dnet:countries @=@ dnet:countries @=@ CX @=@ Christmas Island +dnet:countries @=@ dnet:countries @=@ CC @=@ Cocos (Keeling) Islands +dnet:countries @=@ dnet:countries @=@ CO @=@ Colombia +dnet:countries @=@ dnet:countries @=@ KM @=@ Comoros +dnet:countries @=@ dnet:countries @=@ CG @=@ Congo +dnet:countries @=@ dnet:countries @=@ CD @=@ Congo (Democratic Republic of) +dnet:countries @=@ dnet:countries @=@ CK @=@ Cook Islands +dnet:countries @=@ dnet:countries @=@ CR @=@ Costa Rica +dnet:countries @=@ dnet:countries @=@ CI @=@ Cote d'Ivoire +dnet:countries @=@ dnet:countries @=@ HR @=@ Croatia +dnet:countries @=@ dnet:countries @=@ CU @=@ Cuba +dnet:countries @=@ dnet:countries @=@ CW @=@ Curaçao +dnet:countries @=@ dnet:countries @=@ CY @=@ Cyprus +dnet:countries @=@ dnet:countries @=@ CZ @=@ Czech Republic +dnet:countries @=@ dnet:countries @=@ DK @=@ Denmark +dnet:countries @=@ dnet:countries @=@ DJ @=@ Djibouti +dnet:countries @=@ dnet:countries @=@ DM @=@ Dominica +dnet:countries @=@ dnet:countries @=@ DO @=@ Dominican Republic +dnet:countries @=@ dnet:countries @=@ EC @=@ Ecuador +dnet:countries @=@ dnet:countries @=@ EG @=@ Egypt +dnet:countries @=@ dnet:countries @=@ SV @=@ El Salvador +dnet:countries @=@ dnet:countries @=@ EN @=@ England +dnet:countries @=@ dnet:countries @=@ GQ @=@ Equatorial Guinea +dnet:countries @=@ dnet:countries @=@ ER @=@ Eritrea +dnet:countries @=@ dnet:countries @=@ EE @=@ Estonia +dnet:countries @=@ dnet:countries @=@ ET @=@ Ethiopia +dnet:countries @=@ dnet:countries @=@ EU @=@ European Union +dnet:countries @=@ dnet:countries @=@ FK @=@ Falkland Islands (Malvinas) +dnet:countries @=@ dnet:countries @=@ FO @=@ Faroe Islands +dnet:countries @=@ dnet:countries @=@ FJ @=@ Fiji +dnet:countries @=@ dnet:countries @=@ FI @=@ Finland +dnet:countries @=@ dnet:countries @=@ MK @=@ Former Yugoslav Republic of Macedonia +dnet:countries @=@ dnet:countries @=@ FR @=@ France +dnet:countries @=@ dnet:countries @=@ GF @=@ French Guiana +dnet:countries @=@ dnet:countries @=@ PF @=@ French Polynesia +dnet:countries @=@ dnet:countries @=@ TF @=@ French Southern Territories +dnet:countries @=@ dnet:countries @=@ GA @=@ Gabon +dnet:countries @=@ dnet:countries @=@ GM @=@ Gambia +dnet:countries @=@ dnet:countries @=@ GE @=@ Georgia +dnet:countries @=@ dnet:countries @=@ DE @=@ Germany +dnet:countries @=@ dnet:countries @=@ GH @=@ Ghana +dnet:countries @=@ dnet:countries @=@ GI @=@ Gibraltar +dnet:countries @=@ dnet:countries @=@ GR @=@ Greece +dnet:countries @=@ dnet:countries @=@ GL @=@ Greenland +dnet:countries @=@ dnet:countries @=@ GD @=@ Grenada +dnet:countries @=@ dnet:countries @=@ GP @=@ Guadeloupe +dnet:countries @=@ dnet:countries @=@ GU @=@ Guam +dnet:countries @=@ dnet:countries @=@ GT @=@ Guatemala +dnet:countries @=@ dnet:countries @=@ GG @=@ Guernsey +dnet:countries @=@ dnet:countries @=@ GN @=@ Guinea +dnet:countries @=@ dnet:countries @=@ GW @=@ Guinea-Bissau +dnet:countries @=@ dnet:countries @=@ GY @=@ Guyana +dnet:countries @=@ dnet:countries @=@ HT @=@ Haiti +dnet:countries @=@ dnet:countries @=@ HM @=@ Heard Island and McDonald Islands +dnet:countries @=@ dnet:countries @=@ VA @=@ Holy See (Vatican City State) +dnet:countries @=@ dnet:countries @=@ HN @=@ Honduras +dnet:countries @=@ dnet:countries @=@ HK @=@ Hong Kong +dnet:countries @=@ dnet:countries @=@ HU @=@ Hungary +dnet:countries @=@ dnet:countries @=@ IS @=@ Iceland +dnet:countries @=@ dnet:countries @=@ IN @=@ India +dnet:countries @=@ dnet:countries @=@ ID @=@ Indonesia +dnet:countries @=@ dnet:countries @=@ IR @=@ Iran (Islamic Republic of) +dnet:countries @=@ dnet:countries @=@ IQ @=@ Iraq +dnet:countries @=@ dnet:countries @=@ IE @=@ Ireland +dnet:countries @=@ dnet:countries @=@ IM @=@ Isle of Man +dnet:countries @=@ dnet:countries @=@ IL @=@ Israel +dnet:countries @=@ dnet:countries @=@ IT @=@ Italy +dnet:countries @=@ dnet:countries @=@ JM @=@ Jamaica +dnet:countries @=@ dnet:countries @=@ JP @=@ Japan +dnet:countries @=@ dnet:countries @=@ JE @=@ Jersey +dnet:countries @=@ dnet:countries @=@ JO @=@ Jordan +dnet:countries @=@ dnet:countries @=@ KZ @=@ Kazakhstan +dnet:countries @=@ dnet:countries @=@ KE @=@ Kenya +dnet:countries @=@ dnet:countries @=@ KI @=@ Kiribati +dnet:countries @=@ dnet:countries @=@ KR @=@ Korea (Republic of) +dnet:countries @=@ dnet:countries @=@ KP @=@ Korea, Democatric People's Republic of +dnet:countries @=@ dnet:countries @=@ XK @=@ Kosovo * UN resolution +dnet:countries @=@ dnet:countries @=@ KW @=@ Kuwait +dnet:countries @=@ dnet:countries @=@ KG @=@ Kyrgyzstan +dnet:countries @=@ dnet:countries @=@ LA @=@ Lao (People's Democratic Republic) +dnet:countries @=@ dnet:countries @=@ LV @=@ Latvia +dnet:countries @=@ dnet:countries @=@ LB @=@ Lebanon +dnet:countries @=@ dnet:countries @=@ LS @=@ Lesotho +dnet:countries @=@ dnet:countries @=@ LR @=@ Liberia +dnet:countries @=@ dnet:countries @=@ LY @=@ Libyan Arab Jamahiriya +dnet:countries @=@ dnet:countries @=@ LI @=@ Liechtenstein +dnet:countries @=@ dnet:countries @=@ LT @=@ Lithuania +dnet:countries @=@ dnet:countries @=@ LU @=@ Luxembourg +dnet:countries @=@ dnet:countries @=@ MO @=@ Macao +dnet:countries @=@ dnet:countries @=@ MG @=@ Madagascar +dnet:countries @=@ dnet:countries @=@ MW @=@ Malawi +dnet:countries @=@ dnet:countries @=@ MY @=@ Malaysia +dnet:countries @=@ dnet:countries @=@ MV @=@ Maldives +dnet:countries @=@ dnet:countries @=@ ML @=@ Mali +dnet:countries @=@ dnet:countries @=@ MT @=@ Malta +dnet:countries @=@ dnet:countries @=@ MH @=@ Marshall Islands +dnet:countries @=@ dnet:countries @=@ MQ @=@ Martinique +dnet:countries @=@ dnet:countries @=@ MR @=@ Mauritania +dnet:countries @=@ dnet:countries @=@ MU @=@ Mauritius +dnet:countries @=@ dnet:countries @=@ YT @=@ Mayotte +dnet:countries @=@ dnet:countries @=@ MX @=@ Mexico +dnet:countries @=@ dnet:countries @=@ FM @=@ Micronesia, Federated States of +dnet:countries @=@ dnet:countries @=@ MD @=@ Moldova (Republic of) +dnet:countries @=@ dnet:countries @=@ MN @=@ Mongolia +dnet:countries @=@ dnet:countries @=@ ME @=@ Montenegro +dnet:countries @=@ dnet:countries @=@ MS @=@ Montserrat +dnet:countries @=@ dnet:countries @=@ MA @=@ Morocco +dnet:countries @=@ dnet:countries @=@ MZ @=@ Mozambique +dnet:countries @=@ dnet:countries @=@ MM @=@ Myanmar +dnet:countries @=@ dnet:countries @=@ NA @=@ Namibia +dnet:countries @=@ dnet:countries @=@ NR @=@ Nauru +dnet:countries @=@ dnet:countries @=@ NP @=@ Nepal +dnet:countries @=@ dnet:countries @=@ NL @=@ Netherlands +dnet:countries @=@ dnet:countries @=@ AN @=@ Netherlands Antilles +dnet:countries @=@ dnet:countries @=@ NC @=@ New Caledonia +dnet:countries @=@ dnet:countries @=@ NZ @=@ New Zealand +dnet:countries @=@ dnet:countries @=@ NI @=@ Nicaragua +dnet:countries @=@ dnet:countries @=@ NE @=@ Niger +dnet:countries @=@ dnet:countries @=@ NG @=@ Nigeria +dnet:countries @=@ dnet:countries @=@ NU @=@ Niue +dnet:countries @=@ dnet:countries @=@ NF @=@ Norfolk Island +dnet:countries @=@ dnet:countries @=@ MP @=@ Northern Mariana Islands +dnet:countries @=@ dnet:countries @=@ NO @=@ Norway +dnet:countries @=@ dnet:countries @=@ OC @=@ Oceania +dnet:countries @=@ dnet:countries @=@ OM @=@ Oman +dnet:countries @=@ dnet:countries @=@ PK @=@ Pakistan +dnet:countries @=@ dnet:countries @=@ PW @=@ Palau +dnet:countries @=@ dnet:countries @=@ PS @=@ Palestinian-administered areas +dnet:countries @=@ dnet:countries @=@ PA @=@ Panama +dnet:countries @=@ dnet:countries @=@ PG @=@ Papua New Guinea +dnet:countries @=@ dnet:countries @=@ PY @=@ Paraguay +dnet:countries @=@ dnet:countries @=@ PE @=@ Peru +dnet:countries @=@ dnet:countries @=@ PH @=@ Philippines +dnet:countries @=@ dnet:countries @=@ PN @=@ Pitcairn +dnet:countries @=@ dnet:countries @=@ PL @=@ Poland +dnet:countries @=@ dnet:countries @=@ PT @=@ Portugal +dnet:countries @=@ dnet:countries @=@ PR @=@ Puerto Rico +dnet:countries @=@ dnet:countries @=@ QA @=@ Qatar +dnet:countries @=@ dnet:countries @=@ RO @=@ Romania +dnet:countries @=@ dnet:countries @=@ RU @=@ Russian Federation +dnet:countries @=@ dnet:countries @=@ RW @=@ Rwanda +dnet:countries @=@ dnet:countries @=@ RE @=@ Réunion +dnet:countries @=@ dnet:countries @=@ SH @=@ Saint Helena, Ascension and Tristan da Cunha +dnet:countries @=@ dnet:countries @=@ KN @=@ Saint Kitts and Nevis +dnet:countries @=@ dnet:countries @=@ LC @=@ Saint Lucia +dnet:countries @=@ dnet:countries @=@ MF @=@ Saint Martin (French Part) +dnet:countries @=@ dnet:countries @=@ PM @=@ Saint Pierre and Miquelon +dnet:countries @=@ dnet:countries @=@ VC @=@ Saint Vincent and the Grenadines +dnet:countries @=@ dnet:countries @=@ BL @=@ Saint-Barthélemy +dnet:countries @=@ dnet:countries @=@ WS @=@ Samoa +dnet:countries @=@ dnet:countries @=@ SM @=@ San Marino +dnet:countries @=@ dnet:countries @=@ SA @=@ Saudi Arabia +dnet:countries @=@ dnet:countries @=@ SN @=@ Senegal +dnet:countries @=@ dnet:countries @=@ RS @=@ Serbia +dnet:countries @=@ dnet:countries @=@ SC @=@ Seychelles +dnet:countries @=@ dnet:countries @=@ SL @=@ Sierra Leone +dnet:countries @=@ dnet:countries @=@ SG @=@ Singapore +dnet:countries @=@ dnet:countries @=@ SX @=@ Sint Maarten (Dutch Part) +dnet:countries @=@ dnet:countries @=@ SK @=@ Slovakia +dnet:countries @=@ dnet:countries @=@ SI @=@ Slovenia +dnet:countries @=@ dnet:countries @=@ SB @=@ Solomon Islands +dnet:countries @=@ dnet:countries @=@ SO @=@ Somalia +dnet:countries @=@ dnet:countries @=@ ZA @=@ South Africa +dnet:countries @=@ dnet:countries @=@ GS @=@ South Georgia and the South Sandwich Islands +dnet:countries @=@ dnet:countries @=@ SS @=@ South Sudan +dnet:countries @=@ dnet:countries @=@ ES @=@ Spain +dnet:countries @=@ dnet:countries @=@ LK @=@ Sri Lanka +dnet:countries @=@ dnet:countries @=@ SD @=@ Sudan +dnet:countries @=@ dnet:countries @=@ SR @=@ Suriname +dnet:countries @=@ dnet:countries @=@ SJ @=@ Svalbard and Jan Mayen +dnet:countries @=@ dnet:countries @=@ SZ @=@ Swaziland +dnet:countries @=@ dnet:countries @=@ SE @=@ Sweden +dnet:countries @=@ dnet:countries @=@ CH @=@ Switzerland +dnet:countries @=@ dnet:countries @=@ SY @=@ Syrian Arab Republic +dnet:countries @=@ dnet:countries @=@ ST @=@ São Tomé and Príncipe +dnet:countries @=@ dnet:countries @=@ TW @=@ Taiwan +dnet:countries @=@ dnet:countries @=@ TJ @=@ Tajikistan +dnet:countries @=@ dnet:countries @=@ TZ @=@ Tanzania (United Republic of) +dnet:countries @=@ dnet:countries @=@ TH @=@ Thailand +dnet:countries @=@ dnet:countries @=@ TL @=@ Timor-Leste +dnet:countries @=@ dnet:countries @=@ TG @=@ Togo +dnet:countries @=@ dnet:countries @=@ TK @=@ Tokelau +dnet:countries @=@ dnet:countries @=@ TO @=@ Tonga +dnet:countries @=@ dnet:countries @=@ TT @=@ Trinidad and Tobago +dnet:countries @=@ dnet:countries @=@ TN @=@ Tunisia +dnet:countries @=@ dnet:countries @=@ TR @=@ Turkey +dnet:countries @=@ dnet:countries @=@ TM @=@ Turkmenistan +dnet:countries @=@ dnet:countries @=@ TC @=@ Turks and Caicos Islands +dnet:countries @=@ dnet:countries @=@ TV @=@ Tuvalu +dnet:countries @=@ dnet:countries @=@ UNKNOWN @=@ UNKNOWN +dnet:countries @=@ dnet:countries @=@ UG @=@ Uganda +dnet:countries @=@ dnet:countries @=@ UA @=@ Ukraine +dnet:countries @=@ dnet:countries @=@ AE @=@ United Arab Emirates +dnet:countries @=@ dnet:countries @=@ GB @=@ United Kingdom +dnet:countries @=@ dnet:countries @=@ US @=@ United States +dnet:countries @=@ dnet:countries @=@ UM @=@ United States Minor Outlying Islands +dnet:countries @=@ dnet:countries @=@ UY @=@ Uruguay +dnet:countries @=@ dnet:countries @=@ UZ @=@ Uzbekistan +dnet:countries @=@ dnet:countries @=@ VU @=@ Vanuatu +dnet:countries @=@ dnet:countries @=@ VE @=@ Venezuela +dnet:countries @=@ dnet:countries @=@ VN @=@ Viet Nam +dnet:countries @=@ dnet:countries @=@ VG @=@ Virgin Islands (British) +dnet:countries @=@ dnet:countries @=@ VI @=@ Virgin Islands, U.S. +dnet:countries @=@ dnet:countries @=@ WF @=@ Wallis and Futuna +dnet:countries @=@ dnet:countries @=@ EH @=@ Western Sahara +dnet:countries @=@ dnet:countries @=@ YE @=@ Yemen +dnet:countries @=@ dnet:countries @=@ YU @=@ Yugoslavia +dnet:countries @=@ dnet:countries @=@ ZM @=@ Zambia +dnet:countries @=@ dnet:countries @=@ ZW @=@ Zimbabwe +dnet:countries @=@ dnet:countries @=@ AX @=@ Åland Islands +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasframeworkprogram @=@ hasframeworkprogram +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasprogram @=@ hasprogram +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasspecificprogram @=@ hasspecificprogram +dnet:dataCite_date @=@ dnet:dataCite_date @=@ available @=@ available +dnet:dataCite_date @=@ dnet:dataCite_date @=@ copyrighted @=@ copyrighted +dnet:dataCite_date @=@ dnet:dataCite_date @=@ created @=@ created +dnet:dataCite_date @=@ dnet:dataCite_date @=@ endDate @=@ endDate +dnet:dataCite_date @=@ dnet:dataCite_date @=@ issued @=@ issued +dnet:dataCite_date @=@ dnet:dataCite_date @=@ startDate @=@ startDate +dnet:dataCite_date @=@ dnet:dataCite_date @=@ submitted @=@ submitted +dnet:dataCite_date @=@ dnet:dataCite_date @=@ UNKNOWN @=@ UNKNOWN +dnet:dataCite_date @=@ dnet:dataCite_date @=@ updated @=@ updated +dnet:dataCite_date @=@ dnet:dataCite_date @=@ valid @=@ valid +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire2.0 @=@ OpenAIRE 2.0 (EC funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ driver-openaire2.0 @=@ OpenAIRE 2.0+ (DRIVER OA, EC funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire3.0 @=@ OpenAIRE 3.0 (OA, funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ driver @=@ OpenAIRE Basic (DRIVER OA) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire-cris_1.1 @=@ OpenAIRE CRIS v1.1 +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire2.0_data @=@ OpenAIRE Data (funded, referenced datasets) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire-pub_4.0 @=@ OpenAIRE PubRepos v4.0 +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ hostedBy @=@ collected from a compatible aggregator +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ UNKNOWN @=@ not available +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ native @=@ proprietary +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ notCompatible @=@ under validation +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ accessionNumber @=@ accessionNumber +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ software @=@ software +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ dataset @=@ dataset +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ 171 @=@ Article 171 of the Treaty +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ BSG @=@ Research for the benefit of specific groups +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CIP-EIP-TN @=@ CIP-Eco-Innovation - CIP-Thematic Network +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CP @=@ Collaborative project +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CP-CSA @=@ Combination of CP & CSA +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CSA @=@ Coordination and support action +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ ERC @=@ Support for frontier research (ERC) +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ MC @=@ Support for training and career development of researchers (Marie Curie) +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ NoE @=@ Network of Excellence +fct:funding_relations @=@ fct:funding_relations @=@ fct:hasParentFunding @=@ fct:hasParentFunding +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-ADG @=@ Advanced Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-CSA @=@ Bio-based Industries Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-IA-DEMO @=@ Bio-based Industries Innovation action - Demonstration +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-IA-FLAG @=@ Bio-based Industries Innovation action - Flagship +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-RIA @=@ Bio-based Industries Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-CAR @=@ CAR – Career Restart panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-EJP @=@ COFUND (European Joint Programme) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-PCP @=@ COFUND (PCP) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-PPI @=@ COFUND (PPI) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-CSA @=@ CS2 Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-IA @=@ CS2 Innovation Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-RIA @=@ CS2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CSA-LS @=@ CSA Lump sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-COG @=@ Consolidator Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-CSA @=@ Coordination & support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CSA @=@ Coordination and support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-COFUND-DP @=@ Doctoral programmes +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-CSA @=@ ECSEL Coordination & Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-IA @=@ ECSEL Innovation Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-RIA @=@ ECSEL Research and Innovation Actions +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERA-NET-Cofund @=@ ERA-NET Cofund +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-POC-LS @=@ ERC Proof of Concept Lump Sum Pilot +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-LVG @=@ ERC low value grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ H2020-EEN-SGA @=@ Enterprise Europe Network - Specific Grant Agreement +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-EID @=@ European Industrial Doctorates +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-EJD @=@ European Joint Doctorates +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-ETN @=@ European Training Networks +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-IA @=@ FCH2 Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-RIA @=@ FCH2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-COFUND-FP @=@ Fellowship programmes +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-GF @=@ Global Fellowships +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IMI2-CSA @=@ IMI2 Coordination & support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IMI2-RIA @=@ IMI2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-IA-LS @=@ Innovation Action Lump-Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IA-LS @=@ Innovation Action Lump-Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IA @=@ Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-IA @=@ Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ PCP @=@ Pre-Commercial Procurement +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-POC @=@ Proof of Concept Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ PPI @=@ Public Procurement of Innovative Solutions +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-RI @=@ RI – Reintegration panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-RISE @=@ RISE +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-RIA-LS @=@ Research and Innovation Action Lump-Sum| +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-RIA @=@ Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ RIA @=@ Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ RIA-LS @=@ Research and Innovation action Lump Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-CSA @=@ SESAR: Coordination and Support Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-IA @=@ SESAR: Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-RIA @=@ SESAR: Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SGA-RIA @=@ SGA Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-1 @=@ SME instrument phase 1 +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-2 @=@ SME instrument phase 2 +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-CSA @=@ Shift2Rail - Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-SE @=@ Society and Enterprise panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SGA-CSA @=@ Specific Grant agreement and Coordination and Support Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-ST @=@ Standard EF +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-STG @=@ Starting Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-SyG @=@ Synergy grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-SNLS @=@ Grant to identified beneficiary - Coordination and support actions (MSCA-Special Needs lump sum) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-2b @=@ SME Instrument (grant only and blended finance) +dnet:dataCite_title @=@ dnet:dataCite_title @=@ alternative title @=@ alternative title +dnet:dataCite_title @=@ dnet:dataCite_title @=@ main title @=@ main title +dnet:dataCite_title @=@ dnet:dataCite_title @=@ subtitle @=@ subtitle +dnet:dataCite_title @=@ dnet:dataCite_title @=@ translated title @=@ translated title +nsf:contractTypes @=@ NSF Contract Types @=@ BOA/Task Order @=@ BOA/Task Order +nsf:contractTypes @=@ NSF Contract Types @=@ Continuing grant @=@ Continuing grant +nsf:contractTypes @=@ NSF Contract Types @=@ Contract @=@ Contract +nsf:contractTypes @=@ NSF Contract Types @=@ Contract Interagency Agreement @=@ Contract Interagency Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Cooperative Agreement @=@ Cooperative Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Fellowship @=@ Fellowship +nsf:contractTypes @=@ NSF Contract Types @=@ Fixed Price Award @=@ Fixed Price Award +nsf:contractTypes @=@ NSF Contract Types @=@ Interagency Agreement @=@ Interagency Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Intergovernmental Personnel Award @=@ Intergovernmental Personnel Award +nsf:contractTypes @=@ NSF Contract Types @=@ Personnel Agreement @=@ Personnel Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Standard Grant @=@ Standard Grant +nsf:contractTypes @=@ NSF Contract Types @=@ GAA @=@ GAA +wt:funding_relations @=@ wt:funding_relations @=@ wt:hasParentFunding @=@ wt:hasParentFunding +dnet:access_modes @=@ dnet:access_modes @=@ 12MONTHS @=@ 12 Months Embargo +dnet:access_modes @=@ dnet:access_modes @=@ 6MONTHS @=@ 6 Months Embargo +dnet:access_modes @=@ dnet:access_modes @=@ CLOSED @=@ Closed Access +dnet:access_modes @=@ dnet:access_modes @=@ EMBARGO @=@ Embargo +dnet:access_modes @=@ dnet:access_modes @=@ OPEN @=@ Open Access +dnet:access_modes @=@ dnet:access_modes @=@ OPEN SOURCE @=@ Open Source +dnet:access_modes @=@ dnet:access_modes @=@ OTHER @=@ Other +dnet:access_modes @=@ dnet:access_modes @=@ RESTRICTED @=@ Restricted +dnet:access_modes @=@ dnet:access_modes @=@ UNKNOWN @=@ not available +dnet:result_typologies @=@ dnet:result_typologies @=@ dataset @=@ dataset +dnet:result_typologies @=@ dnet:result_typologies @=@ other @=@ other +dnet:result_typologies @=@ dnet:result_typologies @=@ publication @=@ publication +dnet:result_typologies @=@ dnet:result_typologies @=@ software @=@ software +dnet:languages @=@ dnet:languages @=@ abk @=@ Abkhazian +dnet:languages @=@ dnet:languages @=@ ace @=@ Achinese +dnet:languages @=@ dnet:languages @=@ ach @=@ Acoli +dnet:languages @=@ dnet:languages @=@ ada @=@ Adangme +dnet:languages @=@ dnet:languages @=@ aar @=@ Afar +dnet:languages @=@ dnet:languages @=@ afh @=@ Afrihili +dnet:languages @=@ dnet:languages @=@ afr @=@ Afrikaans +dnet:languages @=@ dnet:languages @=@ afa @=@ Afro-Asiatic +dnet:languages @=@ dnet:languages @=@ aka @=@ Akan +dnet:languages @=@ dnet:languages @=@ akk @=@ Akkadian +dnet:languages @=@ dnet:languages @=@ alb/sqi @=@ Albanian +dnet:languages @=@ dnet:languages @=@ ale @=@ Aleut +dnet:languages @=@ dnet:languages @=@ alg @=@ Algonquian languages +dnet:languages @=@ dnet:languages @=@ tut @=@ Altaic +dnet:languages @=@ dnet:languages @=@ amh @=@ Amharic +dnet:languages @=@ dnet:languages @=@ egy @=@ Ancient Egyptian +dnet:languages @=@ dnet:languages @=@ grc @=@ Ancient Greek +dnet:languages @=@ dnet:languages @=@ apa @=@ Apache +dnet:languages @=@ dnet:languages @=@ ara @=@ Arabic +dnet:languages @=@ dnet:languages @=@ arg @=@ Aragonese +dnet:languages @=@ dnet:languages @=@ arc @=@ Aramaic +dnet:languages @=@ dnet:languages @=@ arp @=@ Arapaho +dnet:languages @=@ dnet:languages @=@ arn @=@ Araucanian +dnet:languages @=@ dnet:languages @=@ arw @=@ Arawak +dnet:languages @=@ dnet:languages @=@ arm/hye @=@ Armenian +dnet:languages @=@ dnet:languages @=@ art @=@ Artificial +dnet:languages @=@ dnet:languages @=@ asm @=@ Assamese +dnet:languages @=@ dnet:languages @=@ ath @=@ Athapascan +dnet:languages @=@ dnet:languages @=@ map @=@ Austronesian +dnet:languages @=@ dnet:languages @=@ ina @=@ Auxiliary Language Association) +dnet:languages @=@ dnet:languages @=@ ava @=@ Avaric +dnet:languages @=@ dnet:languages @=@ ave @=@ Avestan +dnet:languages @=@ dnet:languages @=@ awa @=@ Awadhi +dnet:languages @=@ dnet:languages @=@ aym @=@ Aymara +dnet:languages @=@ dnet:languages @=@ aze @=@ Azerbaijani +dnet:languages @=@ dnet:languages @=@ nah @=@ Aztec +dnet:languages @=@ dnet:languages @=@ ban @=@ Balinese +dnet:languages @=@ dnet:languages @=@ bat @=@ Baltic +dnet:languages @=@ dnet:languages @=@ bal @=@ Baluchi +dnet:languages @=@ dnet:languages @=@ bam @=@ Bambara +dnet:languages @=@ dnet:languages @=@ bai @=@ Bamileke +dnet:languages @=@ dnet:languages @=@ bad @=@ Banda +dnet:languages @=@ dnet:languages @=@ bnt @=@ Bantu +dnet:languages @=@ dnet:languages @=@ bas @=@ Basa +dnet:languages @=@ dnet:languages @=@ bak @=@ Bashkir +dnet:languages @=@ dnet:languages @=@ baq/eus @=@ Basque +dnet:languages @=@ dnet:languages @=@ bej @=@ Beja +dnet:languages @=@ dnet:languages @=@ bel @=@ Belarusian +dnet:languages @=@ dnet:languages @=@ bem @=@ Bemba +dnet:languages @=@ dnet:languages @=@ ben @=@ Bengali +dnet:languages @=@ dnet:languages @=@ ber @=@ Berber +dnet:languages @=@ dnet:languages @=@ bho @=@ Bhojpuri +dnet:languages @=@ dnet:languages @=@ bih @=@ Bihari +dnet:languages @=@ dnet:languages @=@ bik @=@ Bikol +dnet:languages @=@ dnet:languages @=@ bin @=@ Bini +dnet:languages @=@ dnet:languages @=@ bis @=@ Bislama +dnet:languages @=@ dnet:languages @=@ nob @=@ Bokmål, Norwegian; Norwegian Bokmål +dnet:languages @=@ dnet:languages @=@ bos @=@ Bosnian +dnet:languages @=@ dnet:languages @=@ bra @=@ Braj +dnet:languages @=@ dnet:languages @=@ bre @=@ Breton +dnet:languages @=@ dnet:languages @=@ bug @=@ Buginese +dnet:languages @=@ dnet:languages @=@ bul @=@ Bulgarian +dnet:languages @=@ dnet:languages @=@ bua @=@ Buriat +dnet:languages @=@ dnet:languages @=@ bur/mya @=@ Burmese +dnet:languages @=@ dnet:languages @=@ cad @=@ Caddo +dnet:languages @=@ dnet:languages @=@ car @=@ Carib +dnet:languages @=@ dnet:languages @=@ cat @=@ Catalan; Valencian +dnet:languages @=@ dnet:languages @=@ cau @=@ Caucasian +dnet:languages @=@ dnet:languages @=@ ceb @=@ Cebuano +dnet:languages @=@ dnet:languages @=@ cel @=@ Celtic +dnet:languages @=@ dnet:languages @=@ cai @=@ Central American Indian +dnet:languages @=@ dnet:languages @=@ chg @=@ Chagatai +dnet:languages @=@ dnet:languages @=@ cha @=@ Chamorro +dnet:languages @=@ dnet:languages @=@ che @=@ Chechen +dnet:languages @=@ dnet:languages @=@ chr @=@ Cherokee +dnet:languages @=@ dnet:languages @=@ nya @=@ Chewa; Chichewa; Nyanja +dnet:languages @=@ dnet:languages @=@ chy @=@ Cheyenne +dnet:languages @=@ dnet:languages @=@ chb @=@ Chibcha +dnet:languages @=@ dnet:languages @=@ chi/zho @=@ Chinese +dnet:languages @=@ dnet:languages @=@ chn @=@ Chinook jargon +dnet:languages @=@ dnet:languages @=@ cho @=@ Choctaw +dnet:languages @=@ dnet:languages @=@ chu @=@ Church Slavic; Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic +dnet:languages @=@ dnet:languages @=@ chv @=@ Chuvash +dnet:languages @=@ dnet:languages @=@ cop @=@ Coptic +dnet:languages @=@ dnet:languages @=@ cor @=@ Cornish +dnet:languages @=@ dnet:languages @=@ cos @=@ Corsican +dnet:languages @=@ dnet:languages @=@ cre @=@ Cree +dnet:languages @=@ dnet:languages @=@ mus @=@ Creek +dnet:languages @=@ dnet:languages @=@ crp @=@ Creoles and Pidgins +dnet:languages @=@ dnet:languages @=@ hrv @=@ Croatian +dnet:languages @=@ dnet:languages @=@ cus @=@ Cushitic +dnet:languages @=@ dnet:languages @=@ ces/cze @=@ Czech +dnet:languages @=@ dnet:languages @=@ dak @=@ Dakota +dnet:languages @=@ dnet:languages @=@ dan @=@ Danish +dnet:languages @=@ dnet:languages @=@ del @=@ Delaware +dnet:languages @=@ dnet:languages @=@ din @=@ Dinka +dnet:languages @=@ dnet:languages @=@ div @=@ Divehi +dnet:languages @=@ dnet:languages @=@ doi @=@ Dogri +dnet:languages @=@ dnet:languages @=@ dra @=@ Dravidian +dnet:languages @=@ dnet:languages @=@ dua @=@ Duala +dnet:languages @=@ dnet:languages @=@ dut/nld @=@ Dutch; Flemish +dnet:languages @=@ dnet:languages @=@ dyu @=@ Dyula +dnet:languages @=@ dnet:languages @=@ dzo @=@ Dzongkha +dnet:languages @=@ dnet:languages @=@ efi @=@ Efik +dnet:languages @=@ dnet:languages @=@ eka @=@ Ekajuk +dnet:languages @=@ dnet:languages @=@ elx @=@ Elamite +dnet:languages @=@ dnet:languages @=@ eng @=@ English +dnet:languages @=@ dnet:languages @=@ cpe @=@ English-based Creoles and Pidgins +dnet:languages @=@ dnet:languages @=@ esk @=@ Eskimo +dnet:languages @=@ dnet:languages @=@ epo @=@ Esperanto +dnet:languages @=@ dnet:languages @=@ est @=@ Estonian +dnet:languages @=@ dnet:languages @=@ ewe @=@ Ewe +dnet:languages @=@ dnet:languages @=@ ewo @=@ Ewondo +dnet:languages @=@ dnet:languages @=@ fan @=@ Fang +dnet:languages @=@ dnet:languages @=@ fat @=@ Fanti +dnet:languages @=@ dnet:languages @=@ fao @=@ Faroese +dnet:languages @=@ dnet:languages @=@ fij @=@ Fijian +dnet:languages @=@ dnet:languages @=@ fin @=@ Finnish +dnet:languages @=@ dnet:languages @=@ fiu @=@ Finno-Ugrian +dnet:languages @=@ dnet:languages @=@ fon @=@ Fon +dnet:languages @=@ dnet:languages @=@ fra/fre @=@ French +dnet:languages @=@ dnet:languages @=@ cpf @=@ French-based Creoles and Pidgins +dnet:languages @=@ dnet:languages @=@ fry @=@ Frisian +dnet:languages @=@ dnet:languages @=@ ful @=@ Fulah +dnet:languages @=@ dnet:languages @=@ gaa @=@ Ga +dnet:languages @=@ dnet:languages @=@ gae/gdh @=@ Gaelic +dnet:languages @=@ dnet:languages @=@ gla @=@ Gaelic; Scottish Gaelic +dnet:languages @=@ dnet:languages @=@ glg @=@ Galician +dnet:languages @=@ dnet:languages @=@ lug @=@ Ganda +dnet:languages @=@ dnet:languages @=@ gay @=@ Gayo +dnet:languages @=@ dnet:languages @=@ gez @=@ Geez +dnet:languages @=@ dnet:languages @=@ geo/kat @=@ Georgian +dnet:languages @=@ dnet:languages @=@ deu/ger @=@ German +dnet:languages @=@ dnet:languages @=@ gem @=@ Germanic +dnet:languages @=@ dnet:languages @=@ kik @=@ Gikuyu; Kikuyu +dnet:languages @=@ dnet:languages @=@ gil @=@ Gilbertese +dnet:languages @=@ dnet:languages @=@ gon @=@ Gondi +dnet:languages @=@ dnet:languages @=@ got @=@ Gothic +dnet:languages @=@ dnet:languages @=@ grb @=@ Grebo +dnet:languages @=@ dnet:languages @=@ ell/gre @=@ Greek +dnet:languages @=@ dnet:languages @=@ gre/ell @=@ Greek, Modern (1453-) +dnet:languages @=@ dnet:languages @=@ kal @=@ Greenlandic; Kalaallisut +dnet:languages @=@ dnet:languages @=@ grn @=@ Guarani +dnet:languages @=@ dnet:languages @=@ guj @=@ Gujarati +dnet:languages @=@ dnet:languages @=@ hai @=@ Haida +dnet:languages @=@ dnet:languages @=@ hat @=@ Haitian; Haitian Creole +dnet:languages @=@ dnet:languages @=@ hau @=@ Hausa +dnet:languages @=@ dnet:languages @=@ haw @=@ Hawaiian +dnet:languages @=@ dnet:languages @=@ heb @=@ Hebrew +dnet:languages @=@ dnet:languages @=@ her @=@ Herero +dnet:languages @=@ dnet:languages @=@ hil @=@ Hiligaynon +dnet:languages @=@ dnet:languages @=@ him @=@ Himachali +dnet:languages @=@ dnet:languages @=@ hin @=@ Hindi +dnet:languages @=@ dnet:languages @=@ hmo @=@ Hiri Motu +dnet:languages @=@ dnet:languages @=@ hun @=@ Hungarian +dnet:languages @=@ dnet:languages @=@ hup @=@ Hupa +dnet:languages @=@ dnet:languages @=@ iba @=@ Iban +dnet:languages @=@ dnet:languages @=@ ice/isl @=@ Icelandic +dnet:languages @=@ dnet:languages @=@ ido @=@ Ido +dnet:languages @=@ dnet:languages @=@ ibo @=@ Igbo +dnet:languages @=@ dnet:languages @=@ ijo @=@ Ijo +dnet:languages @=@ dnet:languages @=@ ilo @=@ Iloko +dnet:languages @=@ dnet:languages @=@ inc @=@ Indic +dnet:languages @=@ dnet:languages @=@ ine @=@ Indo-European +dnet:languages @=@ dnet:languages @=@ ind @=@ Indonesian +dnet:languages @=@ dnet:languages @=@ ile @=@ Interlingue +dnet:languages @=@ dnet:languages @=@ iku @=@ Inuktitut +dnet:languages @=@ dnet:languages @=@ ipk @=@ Inupiaq +dnet:languages @=@ dnet:languages @=@ ira @=@ Iranian +dnet:languages @=@ dnet:languages @=@ gai/iri @=@ Irish +dnet:languages @=@ dnet:languages @=@ iro @=@ Iroquoian +dnet:languages @=@ dnet:languages @=@ ita @=@ Italian +dnet:languages @=@ dnet:languages @=@ jpn @=@ Japanese +dnet:languages @=@ dnet:languages @=@ jav @=@ Javanese +dnet:languages @=@ dnet:languages @=@ jrb @=@ Judeo-Arabic +dnet:languages @=@ dnet:languages @=@ jpr @=@ Judeo-Persian +dnet:languages @=@ dnet:languages @=@ kab @=@ Kabyle +dnet:languages @=@ dnet:languages @=@ kac @=@ Kachin +dnet:languages @=@ dnet:languages @=@ kam @=@ Kamba +dnet:languages @=@ dnet:languages @=@ kan @=@ Kannada +dnet:languages @=@ dnet:languages @=@ kau @=@ Kanuri +dnet:languages @=@ dnet:languages @=@ kaa @=@ Kara-Kalpak +dnet:languages @=@ dnet:languages @=@ kar @=@ Karen +dnet:languages @=@ dnet:languages @=@ kas @=@ Kashmiri +dnet:languages @=@ dnet:languages @=@ kaw @=@ Kawi +dnet:languages @=@ dnet:languages @=@ kaz @=@ Kazakh +dnet:languages @=@ dnet:languages @=@ kha @=@ Khasi +dnet:languages @=@ dnet:languages @=@ khm @=@ Khmer +dnet:languages @=@ dnet:languages @=@ khi @=@ Khoisan +dnet:languages @=@ dnet:languages @=@ kho @=@ Khotanese +dnet:languages @=@ dnet:languages @=@ kin @=@ Kinyarwanda +dnet:languages @=@ dnet:languages @=@ kir @=@ Kirghiz +dnet:languages @=@ dnet:languages @=@ kom @=@ Komi +dnet:languages @=@ dnet:languages @=@ kon @=@ Kongo +dnet:languages @=@ dnet:languages @=@ kok @=@ Konkani +dnet:languages @=@ dnet:languages @=@ kor @=@ Korean +dnet:languages @=@ dnet:languages @=@ kpe @=@ Kpelle +dnet:languages @=@ dnet:languages @=@ kro @=@ Kru +dnet:languages @=@ dnet:languages @=@ kua @=@ Kuanyama; Kwanyama +dnet:languages @=@ dnet:languages @=@ kum @=@ Kumyk +dnet:languages @=@ dnet:languages @=@ kur @=@ Kurdish +dnet:languages @=@ dnet:languages @=@ kru @=@ Kurukh +dnet:languages @=@ dnet:languages @=@ kus @=@ Kusaie +dnet:languages @=@ dnet:languages @=@ kut @=@ Kutenai +dnet:languages @=@ dnet:languages @=@ lad @=@ Ladino +dnet:languages @=@ dnet:languages @=@ lah @=@ Lahnda +dnet:languages @=@ dnet:languages @=@ lam @=@ Lamba +dnet:languages @=@ dnet:languages @=@ lao @=@ Lao +dnet:languages @=@ dnet:languages @=@ lat @=@ Latin +dnet:languages @=@ dnet:languages @=@ lav @=@ Latvian +dnet:languages @=@ dnet:languages @=@ ltz @=@ Letzeburgesch; Luxembourgish +dnet:languages @=@ dnet:languages @=@ lez @=@ Lezghian +dnet:languages @=@ dnet:languages @=@ lim @=@ Limburgan; Limburger; Limburgish +dnet:languages @=@ dnet:languages @=@ lin @=@ Lingala +dnet:languages @=@ dnet:languages @=@ lit @=@ Lithuanian +dnet:languages @=@ dnet:languages @=@ loz @=@ Lozi +dnet:languages @=@ dnet:languages @=@ lub @=@ Luba-Katanga +dnet:languages @=@ dnet:languages @=@ lui @=@ Luiseno +dnet:languages @=@ dnet:languages @=@ lun @=@ Lunda +dnet:languages @=@ dnet:languages @=@ luo @=@ Luo +dnet:languages @=@ dnet:languages @=@ mac/mak @=@ Macedonian +dnet:languages @=@ dnet:languages @=@ mad @=@ Madurese +dnet:languages @=@ dnet:languages @=@ mag @=@ Magahi +dnet:languages @=@ dnet:languages @=@ mai @=@ Maithili +dnet:languages @=@ dnet:languages @=@ mak @=@ Makasar +dnet:languages @=@ dnet:languages @=@ mlg @=@ Malagasy +dnet:languages @=@ dnet:languages @=@ may/msa @=@ Malay +dnet:languages @=@ dnet:languages @=@ mal @=@ Malayalam +dnet:languages @=@ dnet:languages @=@ mlt @=@ Maltese +dnet:languages @=@ dnet:languages @=@ man @=@ Mandingo +dnet:languages @=@ dnet:languages @=@ mni @=@ Manipuri +dnet:languages @=@ dnet:languages @=@ mno @=@ Manobo +dnet:languages @=@ dnet:languages @=@ glv @=@ Manx +dnet:languages @=@ dnet:languages @=@ mao/mri @=@ Maori +dnet:languages @=@ dnet:languages @=@ mar @=@ Marathi +dnet:languages @=@ dnet:languages @=@ chm @=@ Mari +dnet:languages @=@ dnet:languages @=@ mah @=@ Marshallese +dnet:languages @=@ dnet:languages @=@ mwr @=@ Marwari +dnet:languages @=@ dnet:languages @=@ mas @=@ Masai +dnet:languages @=@ dnet:languages @=@ myn @=@ Mayan +dnet:languages @=@ dnet:languages @=@ men @=@ Mende +dnet:languages @=@ dnet:languages @=@ mic @=@ Micmac +dnet:languages @=@ dnet:languages @=@ dum @=@ Middle Dutch +dnet:languages @=@ dnet:languages @=@ enm @=@ Middle English +dnet:languages @=@ dnet:languages @=@ frm @=@ Middle French +dnet:languages @=@ dnet:languages @=@ gmh @=@ Middle High German +dnet:languages @=@ dnet:languages @=@ mga @=@ Middle Irish +dnet:languages @=@ dnet:languages @=@ min @=@ Minangkabau +dnet:languages @=@ dnet:languages @=@ mis @=@ Miscellaneous +dnet:languages @=@ dnet:languages @=@ moh @=@ Mohawk +dnet:languages @=@ dnet:languages @=@ mol @=@ Moldavian +dnet:languages @=@ dnet:languages @=@ mkh @=@ Mon-Kmer +dnet:languages @=@ dnet:languages @=@ lol @=@ Mongo +dnet:languages @=@ dnet:languages @=@ mon @=@ Mongolian +dnet:languages @=@ dnet:languages @=@ mos @=@ Mossi +dnet:languages @=@ dnet:languages @=@ mul @=@ Multiple languages +dnet:languages @=@ dnet:languages @=@ mun @=@ Munda +dnet:languages @=@ dnet:languages @=@ nau @=@ Nauru +dnet:languages @=@ dnet:languages @=@ nav @=@ Navajo; Navaho +dnet:languages @=@ dnet:languages @=@ nde @=@ Ndebele, North +dnet:languages @=@ dnet:languages @=@ nbl @=@ Ndebele, South +dnet:languages @=@ dnet:languages @=@ ndo @=@ Ndonga +dnet:languages @=@ dnet:languages @=@ nep @=@ Nepali +dnet:languages @=@ dnet:languages @=@ new @=@ Newari +dnet:languages @=@ dnet:languages @=@ nic @=@ Niger-Kordofanian +dnet:languages @=@ dnet:languages @=@ ssa @=@ Nilo-Saharan +dnet:languages @=@ dnet:languages @=@ niu @=@ Niuean +dnet:languages @=@ dnet:languages @=@ non @=@ Norse +dnet:languages @=@ dnet:languages @=@ nai @=@ North American Indian +dnet:languages @=@ dnet:languages @=@ sme @=@ Northern Sami +dnet:languages @=@ dnet:languages @=@ nor @=@ Norwegian +dnet:languages @=@ dnet:languages @=@ nno @=@ Norwegian Nynorsk; Nynorsk, Norwegian +dnet:languages @=@ dnet:languages @=@ nub @=@ Nubian +dnet:languages @=@ dnet:languages @=@ nym @=@ Nyamwezi +dnet:languages @=@ dnet:languages @=@ nyn @=@ Nyankole +dnet:languages @=@ dnet:languages @=@ nyo @=@ Nyoro +dnet:languages @=@ dnet:languages @=@ nzi @=@ Nzima +dnet:languages @=@ dnet:languages @=@ oci @=@ Occitan (post 1500); Provençal +dnet:languages @=@ dnet:languages @=@ oji @=@ Ojibwa +dnet:languages @=@ dnet:languages @=@ ang @=@ Old English +dnet:languages @=@ dnet:languages @=@ fro @=@ Old French +dnet:languages @=@ dnet:languages @=@ goh @=@ Old High German +dnet:languages @=@ dnet:languages @=@ ori @=@ Oriya +dnet:languages @=@ dnet:languages @=@ orm @=@ Oromo +dnet:languages @=@ dnet:languages @=@ osa @=@ Osage +dnet:languages @=@ dnet:languages @=@ oss @=@ Ossetian; Ossetic +dnet:languages @=@ dnet:languages @=@ oto @=@ Otomian +dnet:languages @=@ dnet:languages @=@ ota @=@ Ottoman +dnet:languages @=@ dnet:languages @=@ pal @=@ Pahlavi +dnet:languages @=@ dnet:languages @=@ pau @=@ Palauan +dnet:languages @=@ dnet:languages @=@ pli @=@ Pali +dnet:languages @=@ dnet:languages @=@ pam @=@ Pampanga +dnet:languages @=@ dnet:languages @=@ pag @=@ Pangasinan +dnet:languages @=@ dnet:languages @=@ pan @=@ Panjabi; Punjabi +dnet:languages @=@ dnet:languages @=@ pap @=@ Papiamento +dnet:languages @=@ dnet:languages @=@ paa @=@ Papuan-Australian +dnet:languages @=@ dnet:languages @=@ fas/per @=@ Persian +dnet:languages @=@ dnet:languages @=@ peo @=@ Persian, Old (ca 600 - 400 B.C.) +dnet:languages @=@ dnet:languages @=@ phn @=@ Phoenician +dnet:languages @=@ dnet:languages @=@ pol @=@ Polish +dnet:languages @=@ dnet:languages @=@ pon @=@ Ponape +dnet:languages @=@ dnet:languages @=@ por @=@ Portuguese +dnet:languages @=@ dnet:languages @=@ cpp @=@ Portuguese-based Creoles and Pidgins +dnet:languages @=@ dnet:languages @=@ pra @=@ Prakrit +dnet:languages @=@ dnet:languages @=@ pro @=@ Provencal +dnet:languages @=@ dnet:languages @=@ pus @=@ Pushto +dnet:languages @=@ dnet:languages @=@ que @=@ Quechua +dnet:languages @=@ dnet:languages @=@ roh @=@ Raeto-Romance +dnet:languages @=@ dnet:languages @=@ raj @=@ Rajasthani +dnet:languages @=@ dnet:languages @=@ rar @=@ Rarotongan +dnet:languages @=@ dnet:languages @=@ roa @=@ Romance +dnet:languages @=@ dnet:languages @=@ ron/rum @=@ Romanian +dnet:languages @=@ dnet:languages @=@ rom @=@ Romany +dnet:languages @=@ dnet:languages @=@ run @=@ Rundi +dnet:languages @=@ dnet:languages @=@ rus @=@ Russian +dnet:languages @=@ dnet:languages @=@ sal @=@ Salishan +dnet:languages @=@ dnet:languages @=@ sam @=@ Samaritan +dnet:languages @=@ dnet:languages @=@ smi @=@ Sami +dnet:languages @=@ dnet:languages @=@ smo @=@ Samoan +dnet:languages @=@ dnet:languages @=@ sad @=@ Sandawe +dnet:languages @=@ dnet:languages @=@ sag @=@ Sango +dnet:languages @=@ dnet:languages @=@ san @=@ Sanskrit +dnet:languages @=@ dnet:languages @=@ srd @=@ Sardinian +dnet:languages @=@ dnet:languages @=@ sco @=@ Scots +dnet:languages @=@ dnet:languages @=@ sel @=@ Selkup +dnet:languages @=@ dnet:languages @=@ sem @=@ Semitic +dnet:languages @=@ dnet:languages @=@ srp @=@ Serbian +dnet:languages @=@ dnet:languages @=@ scr @=@ Serbo-Croatian +dnet:languages @=@ dnet:languages @=@ srr @=@ Serer +dnet:languages @=@ dnet:languages @=@ shn @=@ Shan +dnet:languages @=@ dnet:languages @=@ sna @=@ Shona +dnet:languages @=@ dnet:languages @=@ iii @=@ Sichuan Yi +dnet:languages @=@ dnet:languages @=@ sid @=@ Sidamo +dnet:languages @=@ dnet:languages @=@ bla @=@ Siksika +dnet:languages @=@ dnet:languages @=@ snd @=@ Sindhi +dnet:languages @=@ dnet:languages @=@ sin @=@ Sinhala; Sinhalese +dnet:languages @=@ dnet:languages @=@ sit @=@ Sino-Tibetan +dnet:languages @=@ dnet:languages @=@ sio @=@ Siouan +dnet:languages @=@ dnet:languages @=@ sla @=@ Slavic +dnet:languages @=@ dnet:languages @=@ slk/slo @=@ Slovak +dnet:languages @=@ dnet:languages @=@ slv @=@ Slovenian +dnet:languages @=@ dnet:languages @=@ sog @=@ Sogdian +dnet:languages @=@ dnet:languages @=@ som @=@ Somali +dnet:languages @=@ dnet:languages @=@ son @=@ Songhai +dnet:languages @=@ dnet:languages @=@ wen @=@ Sorbian +dnet:languages @=@ dnet:languages @=@ nso @=@ Sotho +dnet:languages @=@ dnet:languages @=@ sot @=@ Sotho, Southern +dnet:languages @=@ dnet:languages @=@ sai @=@ South American Indian +dnet:languages @=@ dnet:languages @=@ esl/spa @=@ Spanish +dnet:languages @=@ dnet:languages @=@ spa @=@ Spanish; Castilian +dnet:languages @=@ dnet:languages @=@ suk @=@ Sukuma +dnet:languages @=@ dnet:languages @=@ sux @=@ Sumerian +dnet:languages @=@ dnet:languages @=@ sun @=@ Sundanese +dnet:languages @=@ dnet:languages @=@ sus @=@ Susu +dnet:languages @=@ dnet:languages @=@ swa @=@ Swahili +dnet:languages @=@ dnet:languages @=@ ssw @=@ Swati +dnet:languages @=@ dnet:languages @=@ swe @=@ Swedish +dnet:languages @=@ dnet:languages @=@ syr @=@ Syriac +dnet:languages @=@ dnet:languages @=@ tgl @=@ Tagalog +dnet:languages @=@ dnet:languages @=@ tah @=@ Tahitian +dnet:languages @=@ dnet:languages @=@ tgk @=@ Tajik +dnet:languages @=@ dnet:languages @=@ tmh @=@ Tamashek +dnet:languages @=@ dnet:languages @=@ tam @=@ Tamil +dnet:languages @=@ dnet:languages @=@ tat @=@ Tatar +dnet:languages @=@ dnet:languages @=@ tel @=@ Telugu +dnet:languages @=@ dnet:languages @=@ ter @=@ Tereno +dnet:languages @=@ dnet:languages @=@ tha @=@ Thai +dnet:languages @=@ dnet:languages @=@ bod/tib @=@ Tibetan +dnet:languages @=@ dnet:languages @=@ tig @=@ Tigre +dnet:languages @=@ dnet:languages @=@ tir @=@ Tigrinya +dnet:languages @=@ dnet:languages @=@ tem @=@ Timne +dnet:languages @=@ dnet:languages @=@ tiv @=@ Tivi +dnet:languages @=@ dnet:languages @=@ tli @=@ Tlingit +dnet:languages @=@ dnet:languages @=@ ton @=@ Tonga (Tonga Islands) +dnet:languages @=@ dnet:languages @=@ tog @=@ Tonga(Nyasa) +dnet:languages @=@ dnet:languages @=@ tru @=@ Truk +dnet:languages @=@ dnet:languages @=@ tsi @=@ Tsimshian +dnet:languages @=@ dnet:languages @=@ tso @=@ Tsonga +dnet:languages @=@ dnet:languages @=@ tsn @=@ Tswana +dnet:languages @=@ dnet:languages @=@ tum @=@ Tumbuka +dnet:languages @=@ dnet:languages @=@ tur @=@ Turkish +dnet:languages @=@ dnet:languages @=@ tuk @=@ Turkmen +dnet:languages @=@ dnet:languages @=@ tyv @=@ Tuvinian +dnet:languages @=@ dnet:languages @=@ twi @=@ Twi +dnet:languages @=@ dnet:languages @=@ uga @=@ Ugaritic +dnet:languages @=@ dnet:languages @=@ uig @=@ Uighur; Uyghur +dnet:languages @=@ dnet:languages @=@ ukr @=@ Ukrainian +dnet:languages @=@ dnet:languages @=@ umb @=@ Umbundu +dnet:languages @=@ dnet:languages @=@ und @=@ Undetermined +dnet:languages @=@ dnet:languages @=@ urd @=@ Urdu +dnet:languages @=@ dnet:languages @=@ uzb @=@ Uzbek +dnet:languages @=@ dnet:languages @=@ vai @=@ Vai +dnet:languages @=@ dnet:languages @=@ ven @=@ Venda +dnet:languages @=@ dnet:languages @=@ vie @=@ Vietnamese +dnet:languages @=@ dnet:languages @=@ vol @=@ Volapük +dnet:languages @=@ dnet:languages @=@ vot @=@ Votic +dnet:languages @=@ dnet:languages @=@ wak @=@ Wakashan +dnet:languages @=@ dnet:languages @=@ wal @=@ Walamo +dnet:languages @=@ dnet:languages @=@ wln @=@ Walloon +dnet:languages @=@ dnet:languages @=@ war @=@ Waray +dnet:languages @=@ dnet:languages @=@ was @=@ Washo +dnet:languages @=@ dnet:languages @=@ cym/wel @=@ Welsh +dnet:languages @=@ dnet:languages @=@ wol @=@ Wolof +dnet:languages @=@ dnet:languages @=@ xho @=@ Xhosa +dnet:languages @=@ dnet:languages @=@ sah @=@ Yakut +dnet:languages @=@ dnet:languages @=@ yao @=@ Yao +dnet:languages @=@ dnet:languages @=@ yap @=@ Yap +dnet:languages @=@ dnet:languages @=@ yid @=@ Yiddish +dnet:languages @=@ dnet:languages @=@ yor @=@ Yoruba +dnet:languages @=@ dnet:languages @=@ zap @=@ Zapotec +dnet:languages @=@ dnet:languages @=@ zen @=@ Zenaga +dnet:languages @=@ dnet:languages @=@ zha @=@ Zhuang; Chuang +dnet:languages @=@ dnet:languages @=@ zul @=@ Zulu +dnet:languages @=@ dnet:languages @=@ zun @=@ Zuni +dnet:languages @=@ dnet:languages @=@ sga @=@ old Irish +dnet:pid_types @=@ dnet:pid_types @=@ GeoPass @=@ Geographic Location-Password Scheme +dnet:pid_types @=@ dnet:pid_types @=@ GBIF @=@ Global Biodiversity Information Facility +dnet:pid_types @=@ dnet:pid_types @=@ ISNI @=@ International Standard Name Identifier +dnet:pid_types @=@ dnet:pid_types @=@ oai @=@ Open Archives Initiative +dnet:pid_types @=@ dnet:pid_types @=@ orcid @=@ Open Researcher and Contributor ID +dnet:pid_types @=@ dnet:pid_types @=@ PANGAEA @=@ PANGAEA +dnet:pid_types @=@ dnet:pid_types @=@ UNKNOWN @=@ UNKNOWN +dnet:pid_types @=@ dnet:pid_types @=@ VIAF @=@ Virtual International Authority File +dnet:pid_types @=@ dnet:pid_types @=@ arXiv @=@ arXiv +dnet:pid_types @=@ dnet:pid_types @=@ doi @=@ doi +dnet:pid_types @=@ dnet:pid_types @=@ info:eu-repo/dai @=@ info:eu-repo/dai +dnet:pid_types @=@ dnet:pid_types @=@ orcidworkid @=@ orcid workid +dnet:pid_types @=@ dnet:pid_types @=@ pmc @=@ pmc +dnet:pid_types @=@ dnet:pid_types @=@ pmid @=@ pmid +dnet:pid_types @=@ dnet:pid_types @=@ urn @=@ urn +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::EuropePMC @=@ file::EuropePMC +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::PDF @=@ file::PDF +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::WoS @=@ file::WoS +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ metadata @=@ metadata +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::hybrid @=@ file::hybrid +fct:funding_typologies @=@ fct:funding_typologies @=@ fct:program @=@ fct:program +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ACM @=@ ACM Computing Classification System +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ agrovoc @=@ AGROVOC +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ bicssc @=@ BIC standard subject categories +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ DFG @=@ DFG Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ddc @=@ Dewey Decimal Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ nsf:fieldOfApplication @=@ Field of Application (NSF) +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ gok @=@ Göttingen Online Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ec:h2020topics @=@ Horizon 2020 Topics +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ jel @=@ JEL Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ lcsh @=@ Library of Congress Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ msc @=@ Mathematics Subject Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ mesheuropmc @=@ Medical Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ mesh @=@ Medical Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ bk @=@ Nederlandse basisclassificatie +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ dnet:od_subjects @=@ OpenDOAR subjects +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ocis @=@ Optics Classification and Indexing Scheme +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ pacs @=@ Physics and Astronomy Classification Scheme +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ rvk @=@ Regensburger Verbundklassifikation +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ UNKNOWN @=@ UNKNOWN +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ wos @=@ Web of Science Subject Areas +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ arxiv @=@ arXiv +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ keyword @=@ keyword +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ udc @=@ Universal Decimal Classification +dnet:protocols @=@ dnet:protocols @=@ HTTPWithFileName @=@ HTTPWithFileName +dnet:protocols @=@ dnet:protocols @=@ NetCDF @=@ NetCDF +dnet:protocols @=@ dnet:protocols @=@ OpenDAP @=@ OpenDAP +dnet:protocols @=@ dnet:protocols @=@ schemaorg @=@ Schema.org +dnet:protocols @=@ dnet:protocols @=@ UNKNOWN @=@ UNKNOWN +dnet:protocols @=@ dnet:protocols @=@ api @=@ api +dnet:protocols @=@ dnet:protocols @=@ dataciteESPlugin @=@ dataciteESPlugin +dnet:protocols @=@ dnet:protocols @=@ datasetsbyjournal @=@ datasetsbyjournal +dnet:protocols @=@ dnet:protocols @=@ datasetsbyproject @=@ datasetsbyproject +dnet:protocols @=@ dnet:protocols @=@ excelFile @=@ excelFile +dnet:protocols @=@ dnet:protocols @=@ file @=@ file +dnet:protocols @=@ dnet:protocols @=@ fileCSV @=@ fileCSV +dnet:protocols @=@ dnet:protocols @=@ fileGzip @=@ fileGzip +dnet:protocols @=@ dnet:protocols @=@ files_by_rpc @=@ files_by_rpc +dnet:protocols @=@ dnet:protocols @=@ files_from_mdstore @=@ files_from_mdstore +dnet:protocols @=@ dnet:protocols @=@ files_from_metadata @=@ files_from_metadata +dnet:protocols @=@ dnet:protocols @=@ filesystem @=@ filesystem +dnet:protocols @=@ dnet:protocols @=@ ftp @=@ ftp +dnet:protocols @=@ dnet:protocols @=@ gristProjects @=@ gristProjects +dnet:protocols @=@ dnet:protocols @=@ gtr2Projects @=@ gtr2Projects +dnet:protocols @=@ dnet:protocols @=@ http @=@ http +dnet:protocols @=@ dnet:protocols @=@ httpCSV @=@ httpCSV +dnet:protocols @=@ dnet:protocols @=@ httpList @=@ httpList +dnet:protocols @=@ dnet:protocols @=@ jdbc @=@ jdbc +dnet:protocols @=@ dnet:protocols @=@ oai @=@ oai +dnet:protocols @=@ dnet:protocols @=@ oai_sets @=@ oai_sets +dnet:protocols @=@ dnet:protocols @=@ other @=@ other +dnet:protocols @=@ dnet:protocols @=@ re3data @=@ re3data +dnet:protocols @=@ dnet:protocols @=@ rest @=@ rest +dnet:protocols @=@ dnet:protocols @=@ rest_json2xml @=@ rest_json2xml +dnet:protocols @=@ dnet:protocols @=@ sftp @=@ sftp +dnet:protocols @=@ dnet:protocols @=@ soap @=@ soap +dnet:protocols @=@ dnet:protocols @=@ sparql @=@ sparql +dnet:protocols @=@ dnet:protocols @=@ sword @=@ sword +dnet:protocols @=@ dnet:protocols @=@ targz @=@ targz +dnet:protocols @=@ dnet:protocols @=@ remoteMdstore @=@ remoteMdstore +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:frameworkprogram @=@ frameworkprogram +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:program @=@ program +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:specificprogram @=@ specificprogram +fct:contractTypes @=@ fct:contractTypes @=@ UNKNOWN @=@ UNKNOWN +wt:contractTypes @=@ wt:contractTypes @=@ UNKNOWN @=@ UNKNOWN +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ UNKNOWN @=@ UNKNOWN +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ collection @=@ collection +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ dataset @=@ dataset +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ event @=@ event +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ film @=@ film +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ image @=@ image +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ interactiveResource @=@ interactiveResource +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ model @=@ model +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ physicalObject @=@ physicalObject +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ service @=@ service +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ software @=@ software +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ sound @=@ sound +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ text @=@ text +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ clinicalTrial @=@ clinical trial +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire2.0 @=@ OpenAIRE 2.0 (EC funding) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire3.0 @=@ OpenAIRE 3.0 (OA, funding) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire4.0 @=@ OpenAIRE 4.0 (inst.&thematic. repo.) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ driver @=@ OpenAIRE Basic (DRIVER OA) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire2.0_data @=@ OpenAIRE Data (funded, referenced datasets) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ hostedBy @=@ collected from a compatible aggregator +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ files @=@ files +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ native @=@ native +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ UNKNOWN @=@ not available +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ notCompatible @=@ under validation +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire-cris_1.1 @=@ OpenAIRE CRIS v1.1 +dnet:publication_resource @=@ dnet:publication_resource @=@ 0018 @=@ Annotation +dnet:publication_resource @=@ dnet:publication_resource @=@ 0001 @=@ Article +dnet:publication_resource @=@ dnet:publication_resource @=@ 0033 @=@ Audiovisual +dnet:publication_resource @=@ dnet:publication_resource @=@ 0008 @=@ Bachelor thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0002 @=@ Book +dnet:publication_resource @=@ dnet:publication_resource @=@ 0037 @=@ Clinical Trial +dnet:publication_resource @=@ dnet:publication_resource @=@ 0022 @=@ Collection +dnet:publication_resource @=@ dnet:publication_resource @=@ 0004 @=@ Conference object +dnet:publication_resource @=@ dnet:publication_resource @=@ 0005 @=@ Contribution for newspaper or weekly magazine +dnet:publication_resource @=@ dnet:publication_resource @=@ 0045 @=@ Data Management Plan +dnet:publication_resource @=@ dnet:publication_resource @=@ 0031 @=@ Data Paper +dnet:publication_resource @=@ dnet:publication_resource @=@ 0021 @=@ Dataset +dnet:publication_resource @=@ dnet:publication_resource @=@ 0006 @=@ Doctoral thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0023 @=@ Event +dnet:publication_resource @=@ dnet:publication_resource @=@ 0009 @=@ External research report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0024 @=@ Film +dnet:publication_resource @=@ dnet:publication_resource @=@ 0025 @=@ Image +dnet:publication_resource @=@ dnet:publication_resource @=@ 0026 @=@ InteractiveResource +dnet:publication_resource @=@ dnet:publication_resource @=@ 0011 @=@ Internal report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0043 @=@ Journal +dnet:publication_resource @=@ dnet:publication_resource @=@ 0010 @=@ Lecture +dnet:publication_resource @=@ dnet:publication_resource @=@ 0007 @=@ Master thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0027 @=@ Model +dnet:publication_resource @=@ dnet:publication_resource @=@ 0012 @=@ Newsletter +dnet:publication_resource @=@ dnet:publication_resource @=@ 0020 @=@ Other ORP type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0039 @=@ Other dataset type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0038 @=@ Other literature type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0040 @=@ Other software type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0013 @=@ Part of book or chapter of book +dnet:publication_resource @=@ dnet:publication_resource @=@ 0019 @=@ Patent +dnet:publication_resource @=@ dnet:publication_resource @=@ 0028 @=@ PhysicalObject +dnet:publication_resource @=@ dnet:publication_resource @=@ 0016 @=@ Preprint +dnet:publication_resource @=@ dnet:publication_resource @=@ 0034 @=@ Project deliverable +dnet:publication_resource @=@ dnet:publication_resource @=@ 0035 @=@ Project milestone +dnet:publication_resource @=@ dnet:publication_resource @=@ 0036 @=@ Project proposal +dnet:publication_resource @=@ dnet:publication_resource @=@ 0017 @=@ Report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0014 @=@ Research +dnet:publication_resource @=@ dnet:publication_resource @=@ 0015 @=@ Review +dnet:publication_resource @=@ dnet:publication_resource @=@ 0029 @=@ Software +dnet:publication_resource @=@ dnet:publication_resource @=@ 0032 @=@ Software Paper +dnet:publication_resource @=@ dnet:publication_resource @=@ 0030 @=@ Sound +dnet:publication_resource @=@ dnet:publication_resource @=@ 0044 @=@ Thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0000 @=@ UNKNOWN +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:aggregator @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:cris @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:datasetarchive @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:entityregistry @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:infospace @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:repository @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:aggregator @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:cris @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:datasetarchive @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:entityregistry @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:infospace @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:repository @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ UNKNOWN @=@ UNKNOWN +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim:pid @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim:search @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:insert @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:actionset @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ iis @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ propagation:project:semrel @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:actionset:orcidworks-no-doi @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:dedup @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk @=@ Harvested +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ crissystem @=@ CRIS System +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ datarepository::unknown @=@ Data Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::datarepository @=@ Data Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ infospace @=@ Information Space +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::institutional @=@ Institutional Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::institutional @=@ Institutional Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::journal @=@ Journal +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::journals @=@ Journal Aggregator/Publisher +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::mock @=@ Other +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubscatalogue::unknown @=@ Publication Catalogue +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::unknown @=@ Publication Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::unknown @=@ Publication Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry @=@ Registry +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ scholarcomminfra @=@ Scholarly Comm. Infrastructure +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::thematic @=@ Thematic Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ websource @=@ Web Source +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::projects @=@ Funder database +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::repositories @=@ Registry of repositories +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ softwarerepository @=@ Software Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::softwarerepository @=@ Software Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ orprepository @=@ Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ researchgraph @=@ Research Graph +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::products @=@ Registry of research products +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::researchers @=@ Registry of researchers +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::organizations @=@ Registry of organizations +wt:funding_typologies @=@ Wellcome Trust: Funding Typologies @=@ wt:fundingStream @=@ Wellcome Trust: Funding Stream +dnet:review_levels @=@ dnet:review_levels @=@ 0000 @=@ UNKNOWN +dnet:review_levels @=@ dnet:review_levels @=@ 0002 @=@ nonPeerReviewed +dnet:review_levels @=@ dnet:review_levels @=@ 0001 @=@ peerReviewed \ No newline at end of file From a2fdf85ba1bacef38c43d94c8f62e7e05ecb265d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 9 Jun 2020 19:52:53 +0200 Subject: [PATCH 51/88] WIP: graph cleaner implementation --- .../dhp/schema/common/ModelConstants.java | 1 + .../eu/dnetlib/dhp/schema/oaf/Instance.java | 6 +-- .../migration/ProtoConverter.java | 11 ++++- .../DnetCollectorWorkerApplicationTests.java | 2 + .../doiboost/crossref/Crossref2Oaf.scala | 6 ++- .../dhp/oa/graph/clean/CleaningRule.java | 16 +++---- .../dhp/oa/graph/raw/OafToOafMapper.java | 10 +--- .../dhp/oa/graph/raw/OdfToOafMapper.java | 16 +------ .../oa/graph/raw/common/VocabularyGroup.java | 18 ++++++-- .../dhp/oa/graph/clean/CleaningRuleTest.java | 40 ++++++++++++++-- .../eu/dnetlib/dhp/oa/graph/clean/result.json | 46 ++++++++++++++++++- .../oa/provision/utils/XmlRecordFactory.java | 4 +- 12 files changed, 131 insertions(+), 45 deletions(-) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java index e32dd10fa..fba20dda1 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java @@ -14,6 +14,7 @@ public class ModelConstants { public static final String DNET_DATA_CITE_RESOURCE = "dnet:dataCite_resource"; public static final String DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions"; public static final String DNET_COUNTRY_TYPE = "dnet:countries"; + public static final String DNET_REVIEW_LEVELS = "dnet:review_levels"; public static final String SYSIMPORT_CROSSWALK_REPOSITORY = "sysimport:crosswalk:repository"; public static final String SYSIMPORT_CROSSWALK_ENTITYREGISTRY = "sysimport:crosswalk:entityregistry"; diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java index 2b7d3846c..29d495261 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java @@ -31,7 +31,7 @@ public class Instance implements Serializable { // typed results private Field processingchargecurrency; - private Field refereed; // peer-review status + private Qualifier refereed; // peer-review status public Field getLicense() { return license; @@ -113,11 +113,11 @@ public class Instance implements Serializable { this.processingchargecurrency = processingchargecurrency; } - public Field getRefereed() { + public Qualifier getRefereed() { return refereed; } - public void setRefereed(Field refereed) { + public void setRefereed(Qualifier refereed) { this.refereed = refereed; } diff --git a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java index e55c0eb7b..8ea877aec 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java +++ b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/migration/ProtoConverter.java @@ -96,12 +96,21 @@ public class ProtoConverter implements Serializable { .stream() .distinct() .collect(Collectors.toCollection(ArrayList::new)) : null); - i.setRefereed(mapStringField(ri.getRefereed())); + i.setRefereed(mapRefereed(ri.getRefereed())); i.setProcessingchargeamount(mapStringField(ri.getProcessingchargeamount())); i.setProcessingchargecurrency(mapStringField(ri.getProcessingchargecurrency())); return i; } + private static Qualifier mapRefereed(FieldTypeProtos.StringField refereed) { + Qualifier q = new Qualifier(); + q.setClassid(refereed.getValue()); + q.setSchemename(refereed.getValue()); + q.setSchemeid("dnet:review_levels"); + q.setSchemename("dnet:review_levels"); + return q; + } + private static List convertExternalRefs(OafProtos.Oaf oaf) { ResultProtos.Result r = oaf.getEntity().getResult(); if (r.getExternalReferenceCount() > 0) { diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collector/worker/DnetCollectorWorkerApplicationTests.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collector/worker/DnetCollectorWorkerApplicationTests.java index 87bd3be3d..c745219fe 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collector/worker/DnetCollectorWorkerApplicationTests.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collector/worker/DnetCollectorWorkerApplicationTests.java @@ -8,6 +8,7 @@ import java.io.File; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import com.fasterxml.jackson.databind.ObjectMapper; @@ -19,6 +20,7 @@ import eu.dnetlib.dhp.collection.worker.utils.CollectorPluginFactory; import eu.dnetlib.message.Message; import eu.dnetlib.message.MessageManager; +@Disabled public class DnetCollectorWorkerApplicationTests { private final ArgumentApplicationParser argumentParser = mock(ArgumentApplicationParser.class); diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index cc2c9d586..ec8aca55c 100644 --- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -166,8 +166,10 @@ case object Crossref2Oaf { val has_review = (json \ "relation" \"has-review" \ "id") - if(has_review != JNothing) - instance.setRefereed(asField("peerReviewed")) + if(has_review != JNothing) { + instance.setRefereed( + createQualifier("0001", "peerReviewed", "dnet:review_levels", "dnet:review_levels")) + } instance.setAccessright(getRestrictedQualifier()) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java index c00c7b4d9..51b930962 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java @@ -1,19 +1,16 @@ package eu.dnetlib.dhp.oa.graph.clean; -import java.beans.BeanInfo; -import java.beans.IntrospectionException; -import java.beans.Introspector; -import java.beans.PropertyDescriptor; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.util.*; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; import org.apache.spark.api.java.function.MapFunction; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.oaf.Oaf; -import eu.dnetlib.dhp.schema.oaf.Publication; import eu.dnetlib.dhp.schema.oaf.Qualifier; public class CleaningRule implements MapFunction { @@ -61,7 +58,6 @@ public class CleaningRule implements MapFunction { if (value instanceof Qualifier) { Qualifier q = (Qualifier) value; if (vocabularies.vocabularyExists(q.getSchemeid())) { - field.set(o, vocabularies.lookup(q.getSchemeid(), q.getClassid())); } @@ -86,4 +82,8 @@ public class CleaningRule implements MapFunction { return fields; } + + public VocabularyGroup getVocabularies() { + return vocabularies; + } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index 6f91ce733..2ea8bba4a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -4,13 +4,7 @@ package eu.dnetlib.dhp.oa.graph.raw; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.createOpenaireId; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.field; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_ACCESS_MODES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_LANGUAGES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PID_TYPES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PUBLICATION_RESOURCE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_RELATED_TO; -import static eu.dnetlib.dhp.schema.common.ModelConstants.PUBLICATION_DATASET; -import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_RESULT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import java.util.ArrayList; import java.util.List; @@ -139,7 +133,7 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { instance .setAccessright(prepareQualifier(doc, "//oaf:accessrights", DNET_ACCESS_MODES)); instance.setLicense(field(doc.valueOf("//oaf:license"), info)); - instance.setRefereed(field(doc.valueOf("//oaf:refereed"), info)); + instance.setRefereed(prepareQualifier(doc, "//oaf:refereed", DNET_REVIEW_LEVELS)); instance .setProcessingchargeamount(field(doc.valueOf("//oaf:processingchargeamount"), info)); instance diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index bbd9442e1..a2019e959 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -4,19 +4,7 @@ package eu.dnetlib.dhp.oa.graph.raw; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.createOpenaireId; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.field; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_ACCESS_MODES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_DATA_CITE_DATE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_DATA_CITE_RESOURCE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_LANGUAGES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PID_TYPES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PUBLICATION_RESOURCE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.HAS_PARTS; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_PART_OF; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_SUPPLEMENTED_BY; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_SUPPLEMENT_TO; -import static eu.dnetlib.dhp.schema.common.ModelConstants.PART; -import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_RESULT; -import static eu.dnetlib.dhp.schema.common.ModelConstants.SUPPLEMENT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import java.util.ArrayList; import java.util.Arrays; @@ -129,7 +117,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { instance .setAccessright(prepareQualifier(doc, "//oaf:accessrights", DNET_ACCESS_MODES)); instance.setLicense(field(doc.valueOf("//oaf:license"), info)); - instance.setRefereed(field(doc.valueOf("//oaf:refereed"), info)); + instance.setRefereed(prepareQualifier(doc, "//oaf:refereed", DNET_REVIEW_LEVELS)); instance.setProcessingchargeamount(field(doc.valueOf("//oaf:processingchargeamount"), info)); instance .setProcessingchargecurrency(field(doc.valueOf("//oaf:processingchargeamount/@currency"), info)); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java index 6af80683e..ec95ade00 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java @@ -2,10 +2,9 @@ package eu.dnetlib.dhp.oa.graph.raw.common; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -86,6 +85,19 @@ public class VocabularyGroup implements Serializable { } } + public Set getTerms(String vocId) { + if (!vocabularyExists(vocId)) { + return new HashSet<>(); + } + return vocs + .get(vocId.toLowerCase()) + .getTerms() + .values() + .stream() + .map(t -> t.getId()) + .collect(Collectors.toCollection(HashSet::new)); + } + public Qualifier lookup(String vocId, String id) { return Optional .ofNullable(getSynonymAsQualifier(vocId, id)) diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java index fab3c0c01..019285cc3 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java @@ -1,14 +1,17 @@ package eu.dnetlib.dhp.oa.graph.clean; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.lenient; import java.io.IOException; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -18,7 +21,9 @@ import org.mockito.junit.jupiter.MockitoExtension; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyTerm; import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Qualifier; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; @@ -26,6 +31,7 @@ import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; public class CleaningRuleTest { public static final ObjectMapper MAPPER = new ObjectMapper(); + @Mock private ISLookUpService isLookUpService; @@ -47,18 +53,46 @@ public class CleaningRuleTest { @Test public void testCleaning() throws Exception { + assertNotNull(cleaningRule.getVocabularies()); + String json = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/result.json")); Publication p_in = MAPPER.readValue(json, Publication.class); Publication p_out = cleaningRule.call(p_in); - Assertions.assertNotNull(p_out); + assertNotNull(p_out); + + assertEquals("eng", p_out.getLanguage().getClassid()); + assertEquals("English", p_out.getLanguage().getClassname()); + + assertEquals("0018", p_out.getInstance().get(0).getInstancetype().getClassid()); + assertEquals("Annotation", p_out.getInstance().get(0).getInstancetype().getClassname()); + + assertEquals("CLOSED", p_out.getInstance().get(0).getAccessright().getClassid()); + assertEquals("Closed Access", p_out.getInstance().get(0).getAccessright().getClassname()); + + Set pidTerms = vocabularies.getTerms("dnet:pid_types"); + assertTrue( + p_out + .getPid() + .stream() + .map(p -> p.getQualifier()) + .allMatch(q -> pidTerms.contains(q.getClassid()))); // TODO add more assertions to verity the cleaned values System.out.println(MAPPER.writeValueAsString(p_out)); } + private Stream getAuthorPidTypes(Publication pub) { + return pub + .getAuthor() + .stream() + .map(a -> a.getPid()) + .flatMap(p -> p.stream()) + .map(s -> s.getQualifier()); + } + private List vocs() throws IOException { return IOUtils .readLines(CleaningRuleTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/terms.txt")); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json index c45544b40..435b001b7 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json @@ -6,6 +6,28 @@ "fullname": "Brien, Tom", "name": "Tom", "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "ORCID12", + "classname": "ORCID12", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "0000-0001-9613-6639" + } ], "rank": 1, "surname": "Brien" @@ -16,6 +38,28 @@ "fullname": "Ade, Peter", "name": "Peter", "pid": [ + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "xyz", + "classname": "XYZ", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "qwerty" + } ], "rank": 2, "surname": "Ade" @@ -207,7 +251,7 @@ { "accessright": { "classid": "CLOSED", - "classname": "Closed Access", + "classname": "CLOSED", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes" }, diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java index d950a816d..cb45f3f32 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java @@ -1165,10 +1165,10 @@ public class XmlRecordFactory implements Serializable { .asXmlElement( "distributionlocation", instance.getDistributionlocation())); } - if (instance.getRefereed() != null && isNotBlank(instance.getRefereed().getValue())) { + if (instance.getRefereed() != null && !instance.getRefereed().isBlank()) { fields .add( - XmlSerializationUtils.asXmlElement("refereed", instance.getRefereed().getValue())); + XmlSerializationUtils.mapQualifier("refereed", instance.getRefereed())); } if (instance.getProcessingchargeamount() != null && isNotBlank(instance.getProcessingchargeamount().getValue())) { From 7177a32d75fbc2badc0cb1fbe9bc806ca24dd52e Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 10 Jun 2020 10:04:00 +0200 Subject: [PATCH 52/88] import of invisible stores --- .../raw/AbstractMdRecordToOafMapper.java | 13 ++++++---- .../raw/GenerateEntitiesApplication.java | 12 ++++++---- .../raw/MigrateMongoMdstoresApplication.java | 5 ++-- .../dhp/oa/graph/raw/OafToOafMapper.java | 4 ++-- .../dhp/oa/graph/raw/OdfToOafMapper.java | 4 ++-- .../oa/graph/raw_all/oozie_app/workflow.xml | 17 +++++++++++++ .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 24 ++++++++++++++++--- 7 files changed, 60 insertions(+), 19 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index ab1e89187..fc77950d0 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -63,6 +63,8 @@ public abstract class AbstractMdRecordToOafMapper { protected final VocabularyGroup vocs; + private final boolean invisible; + protected static final String DATACITE_SCHEMA_KERNEL_4 = "http://datacite.org/schema/kernel-4"; protected static final String DATACITE_SCHEMA_KERNEL_3 = "http://datacite.org/schema/kernel-3"; protected static final Qualifier ORCID_PID_TYPE = qualifier( @@ -85,8 +87,9 @@ public abstract class AbstractMdRecordToOafMapper { protected static final Qualifier MAIN_TITLE_QUALIFIER = qualifier( "main title", "main title", "dnet:dataCite_title", "dnet:dataCite_title"); - protected AbstractMdRecordToOafMapper(final VocabularyGroup vocs) { + protected AbstractMdRecordToOafMapper(final VocabularyGroup vocs, final boolean invisible) { this.vocs = vocs; + this.invisible = invisible; } public List processMdRecord(final String xml) { @@ -112,7 +115,7 @@ public abstract class AbstractMdRecordToOafMapper { return null; } - final DataInfo info = prepareDataInfo(doc); + final DataInfo info = prepareDataInfo(doc, invisible); final long lastUpdateTimestamp = new Date().getTime(); return createOafs(doc, type, collectedFrom, hostedBy, info, lastUpdateTimestamp); @@ -510,11 +513,11 @@ public abstract class AbstractMdRecordToOafMapper { return oaiIProvenance(identifier, baseURL, metadataNamespace, altered, datestamp, harvestDate); } - protected DataInfo prepareDataInfo(final Document doc) { + protected DataInfo prepareDataInfo(final Document doc, final boolean invisible) { final Node n = doc.selectSingleNode("//oaf:datainfo"); if (n == null) { - return dataInfo(false, null, false, false, REPOSITORY_PROVENANCE_ACTIONS, "0.9"); + return dataInfo(false, null, false, invisible, REPOSITORY_PROVENANCE_ACTIONS, "0.9"); } final String paClassId = n.valueOf("./oaf:provenanceaction/@classid"); @@ -528,7 +531,7 @@ public abstract class AbstractMdRecordToOafMapper { final String trust = n.valueOf("./oaf:trust"); return dataInfo( - deletedbyinference, inferenceprovenance, inferred, false, + deletedbyinference, inferenceprovenance, inferred, invisible, qualifier(paClassId, paClassName, paSchemeId, paSchemeName), trust); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java index 8e5ba9cd1..4262732a5 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java @@ -137,10 +137,14 @@ public class GenerateEntitiesApplication { final String type = StringUtils.substringAfter(id, ":"); switch (type.toLowerCase()) { - case "native_oaf": - return new OafToOafMapper(vocs).processMdRecord(s); - case "native_odf": - return new OdfToOafMapper(vocs).processMdRecord(s); + case "oaf-store-cleaned": + return new OafToOafMapper(vocs, false).processMdRecord(s); + case "odf-store-cleaned": + return new OdfToOafMapper(vocs, false).processMdRecord(s); + case "oaf-store-intersection": + return new OafToOafMapper(vocs, true).processMdRecord(s); + case "odf-store-intersection": + return new OdfToOafMapper(vocs, true).processMdRecord(s); case "datasource": return Arrays.asList(convertFromJson(s, Datasource.class)); case "organization": diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java index 00c1dc4bb..e7703bf72 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateMongoMdstoresApplication.java @@ -26,8 +26,7 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrationApplicatio IOUtils .toString( MigrateMongoMdstoresApplication.class - .getResourceAsStream( - "/eu/dnetlib/dhp/oa/graph/migrate_mongo_mstores_parameters.json"))); + .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/migrate_mongo_mstores_parameters.json"))); parser.parseArgument(args); final String mongoBaseUrl = parser.get("mongoBaseUrl"); @@ -60,7 +59,7 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrationApplicatio final String currentColl = entry.getValue(); for (final String xml : mdstoreClient.listRecords(currentColl)) { - emit(xml, "native_" + format); + emit(xml, String.format("%s-%s-%s", format, layout, interpretation)); } } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index 6f91ce733..e44f830df 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -37,8 +37,8 @@ import eu.dnetlib.dhp.schema.oaf.StructuredProperty; public class OafToOafMapper extends AbstractMdRecordToOafMapper { - public OafToOafMapper(final VocabularyGroup vocs) { - super(vocs); + public OafToOafMapper(final VocabularyGroup vocs, final boolean invisible) { + super(vocs, invisible); } @Override diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index bbd9442e1..e6a744fc0 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -44,8 +44,8 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { public static final String HTTP_DX_DOI_PREIFX = "http://dx.doi.org/"; - public OdfToOafMapper(final VocabularyGroup vocs) { - super(vocs); + public OdfToOafMapper(final VocabularyGroup vocs, final boolean invisible) { + super(vocs, invisible); } @Override diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml index c2bea9f8a..8c58bf39a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml @@ -210,6 +210,23 @@ --mdLayoutstore --mdInterpretationcleaned + + + + + + + + + + eu.dnetlib.dhp.oa.graph.raw.MigrateMongoMdstoresApplication + --hdfsPath${contentPath}/oaf_records_invisible + --mongoBaseUrl${mongoURL} + --mongoDb${mongoDb} + --mdFormatOAF + --mdLayoutstore + --mdInterpretationintersection + diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index dad427ce4..9bd20303f 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -2,6 +2,7 @@ package eu.dnetlib.dhp.oa.graph.raw; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; @@ -55,7 +56,7 @@ public class MappersTest { final String xml = IOUtils.toString(getClass().getResourceAsStream("oaf_record.xml")); - final List list = new OafToOafMapper(vocs).processMdRecord(xml); + final List list = new OafToOafMapper(vocs, false).processMdRecord(xml); assertEquals(3, list.size()); assertTrue(list.get(0) instanceof Publication); @@ -69,6 +70,7 @@ public class MappersTest { assertValidId(p.getId()); assertValidId(p.getCollectedfrom().get(0).getKey()); assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue())); + assertFalse(p.getDataInfo().getInvisible()); assertTrue(p.getAuthor().size() > 0); final Optional author = p @@ -134,11 +136,27 @@ public class MappersTest { // System.out.println(new ObjectMapper().writeValueAsString(r2)); } + @Test + void testPublicationInvisible() throws IOException { + + final String xml = IOUtils.toString(getClass().getResourceAsStream("oaf_record.xml")); + + final List list = new OafToOafMapper(vocs, true).processMdRecord(xml); + + assertTrue(list.size() > 0); + assertTrue(list.get(0) instanceof Publication); + + final Publication p = (Publication) list.get(0); + + assertTrue(p.getDataInfo().getInvisible()); + + } + @Test void testDataset() throws IOException { final String xml = IOUtils.toString(getClass().getResourceAsStream("odf_dataset.xml")); - final List list = new OdfToOafMapper(vocs).processMdRecord(xml); + final List list = new OdfToOafMapper(vocs, false).processMdRecord(xml); assertEquals(3, list.size()); assertTrue(list.get(0) instanceof Dataset); @@ -220,7 +238,7 @@ public class MappersTest { void testSoftware() throws IOException { final String xml = IOUtils.toString(getClass().getResourceAsStream("odf_software.xml")); - final List list = new OdfToOafMapper(vocs).processMdRecord(xml); + final List list = new OdfToOafMapper(vocs, false).processMdRecord(xml); assertEquals(1, list.size()); assertTrue(list.get(0) instanceof Software); From c08e66e01e2c5ff0726e835f214699be8bb68ab5 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 10 Jun 2020 10:11:56 +0200 Subject: [PATCH 53/88] fixed a workflow parameter --- .../eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml index 8c58bf39a..9a7e36570 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml @@ -254,7 +254,7 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --sourcePaths${contentPath}/db_claims,${contentPath}/oaf_claims,${contentPath}/odf_claims + --sourcePaths${contentPath}/db_claims,${contentPath}/oaf_claims,${contentPath}/odf_claims,${contentPath}/oaf_records_invisible --targetPath${workingDir}/entities_claim --isLookupUrl${isLookupUrl} From 5869cb76b31bde5f764202fcdcf74f879ce7224c Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 10 Jun 2020 12:11:16 +0200 Subject: [PATCH 54/88] reformatting --- .../broker/oa/GenerateEventsApplication.java | 122 ++++++++++++------ .../dhp/broker/oa/matchers/UpdateMatcher.java | 6 +- .../simple/EnrichMissingAbstract.java | 6 +- .../simple/EnrichMissingAuthorOrcid.java | 3 +- .../simple/EnrichMissingOpenAccess.java | 7 +- .../oa/matchers/simple/EnrichMissingPid.java | 10 +- .../simple/EnrichMissingPublicationDate.java | 6 +- .../matchers/simple/EnrichMissingSubject.java | 3 +- .../matchers/simple/EnrichMoreOpenAccess.java | 3 +- .../oa/matchers/simple/EnrichMorePid.java | 6 +- .../oa/matchers/simple/EnrichMoreSubject.java | 3 +- .../dhp/broker/oa/util/EventGroup.java | 1 + .../dhp/broker/oa/util/ResultAggregator.java | 1 + .../dhp/broker/oa/util/ResultGroup.java | 1 + .../dhp/broker/oa/util/TrustUtils.java | 16 ++- .../dhp/broker/oa/util/UpdateInfo.java | 9 +- .../dhp/broker/oa/util/TrustUtilsTest.java | 1 + 17 files changed, 138 insertions(+), 66 deletions(-) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java index 44bc5cb6e..ecf4e3eff 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java @@ -91,35 +91,29 @@ public class GenerateEventsApplication { private static final UpdateMatcher>, ?> enrichMoreSoftware = new EnrichMoreSoftware(); private static final UpdateMatcher>, ?> enrichMisissingPublicationIsRelatedTo = new EnrichMissingPublicationIsRelatedTo(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsReferencedBy = - new EnrichMissingPublicationIsReferencedBy(); + private static final UpdateMatcher>, ?> enrichMissingPublicationIsReferencedBy = new EnrichMissingPublicationIsReferencedBy(); private static final UpdateMatcher>, ?> enrichMissingPublicationReferences = new EnrichMissingPublicationReferences(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsSupplementedTo = - new EnrichMissingPublicationIsSupplementedTo(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsSupplementedBy = - new EnrichMissingPublicationIsSupplementedBy(); + private static final UpdateMatcher>, ?> enrichMissingPublicationIsSupplementedTo = new EnrichMissingPublicationIsSupplementedTo(); + private static final UpdateMatcher>, ?> enrichMissingPublicationIsSupplementedBy = new EnrichMissingPublicationIsSupplementedBy(); - private static final UpdateMatcher>, ?> enrichMisissingDatasetIsRelatedTo = - new EnrichMissingDatasetIsRelatedTo(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsReferencedBy = - new EnrichMissingDatasetIsReferencedBy(); - private static final UpdateMatcher>, ?> enrichMissingDatasetReferences = - new EnrichMissingDatasetReferences(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsSupplementedTo = - new EnrichMissingDatasetIsSupplementedTo(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsSupplementedBy = - new EnrichMissingDatasetIsSupplementedBy(); + private static final UpdateMatcher>, ?> enrichMisissingDatasetIsRelatedTo = new EnrichMissingDatasetIsRelatedTo(); + private static final UpdateMatcher>, ?> enrichMissingDatasetIsReferencedBy = new EnrichMissingDatasetIsReferencedBy(); + private static final UpdateMatcher>, ?> enrichMissingDatasetReferences = new EnrichMissingDatasetReferences(); + private static final UpdateMatcher>, ?> enrichMissingDatasetIsSupplementedTo = new EnrichMissingDatasetIsSupplementedTo(); + private static final UpdateMatcher>, ?> enrichMissingDatasetIsSupplementedBy = new EnrichMissingDatasetIsSupplementedBy(); // Aggregators - private static final TypedColumn, ResultGroup> resultAggrTypedColumn = new ResultAggregator().toColumn(); + private static final TypedColumn, ResultGroup> resultAggrTypedColumn = new ResultAggregator() + .toColumn(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static void main(final String[] args) throws Exception { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils - .toString(GenerateEventsApplication.class - .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/merge_claims_parameters.json"))); + .toString( + GenerateEventsApplication.class + .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/merge_claims_parameters.json"))); parser.parseArgument(args); final Boolean isSparkSessionManaged = Optional @@ -172,18 +166,23 @@ public class GenerateEventsApplication { final Class resultClazz, final DedupConfig dedupConfig) { - final Dataset results = readPath(spark, graphPath + "/" + resultClazz.getSimpleName().toLowerCase(), Result.class) - .filter(r -> r.getDataInfo().getDeletedbyinference()); + final Dataset results = readPath( + spark, graphPath + "/" + resultClazz.getSimpleName().toLowerCase(), Result.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()); final Dataset mergedRels = readPath(spark, graphPath + "/relation", Relation.class) .filter(r -> r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); - return results.joinWith(mergedRels, results.col("id").equalTo(mergedRels.col("source")), "inner") + return results + .joinWith(mergedRels, results.col("id").equalTo(mergedRels.col("source")), "inner") .groupByKey((MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) .agg(resultAggrTypedColumn) .map((MapFunction, ResultGroup>) t -> t._2, Encoders.kryo(ResultGroup.class)) .filter(ResultGroup::isValid) - .map((MapFunction) g -> GenerateEventsApplication.generateSimpleEvents(g, dedupConfig), Encoders.kryo(EventGroup.class)) + .map( + (MapFunction) g -> GenerateEventsApplication + .generateSimpleEvents(g, dedupConfig), + Encoders.kryo(EventGroup.class)) .flatMap(group -> group.getData().iterator(), Encoders.kryo(Event.class)); } @@ -207,16 +206,19 @@ public class GenerateEventsApplication { return events; } - private static Dataset generateRelationEvents(final SparkSession spark, + private static Dataset generateRelationEvents( + final SparkSession spark, final String graphPath, final Class sourceClass, final Class targetClass, final DedupConfig dedupConfig) { - final Dataset sources = readPath(spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) - .filter(r -> r.getDataInfo().getDeletedbyinference()); + final Dataset sources = readPath( + spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()); - final Dataset targets = readPath(spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), targetClass); + final Dataset targets = readPath( + spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), targetClass); final Dataset mergedRels = readPath(spark, graphPath + "/relation", Relation.class) .filter(r -> r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); @@ -224,7 +226,8 @@ public class GenerateEventsApplication { final Dataset rels = readPath(spark, graphPath + "/relation", Relation.class) .filter(r -> !r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); - final Dataset duplicates = sources.joinWith(mergedRels, sources.col("id").equalTo(rels.col("source")), "inner") + final Dataset duplicates = sources + .joinWith(mergedRels, sources.col("id").equalTo(rels.col("source")), "inner") .groupByKey((MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) .agg(resultAggrTypedColumn) .map((MapFunction, ResultGroup>) t -> t._2, Encoders.kryo(ResultGroup.class)) @@ -243,7 +246,8 @@ public class GenerateEventsApplication { return null; } - private List generateProjectsEvents(final Collection>> childrenWithProjects, final DedupConfig dedupConfig) { + private List generateProjectsEvents(final Collection>> childrenWithProjects, + final DedupConfig dedupConfig) { final List> list = new ArrayList<>(); for (final Pair> target : childrenWithProjects) { @@ -254,7 +258,8 @@ public class GenerateEventsApplication { return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); } - private List generateSoftwareEvents(final Collection>> childrenWithSoftwares, final DedupConfig dedupConfig) { + private List generateSoftwareEvents(final Collection>> childrenWithSoftwares, + final DedupConfig dedupConfig) { final List> list = new ArrayList<>(); for (final Pair> target : childrenWithSoftwares) { @@ -279,15 +284,30 @@ public class GenerateEventsApplication { for (final Pair> target : cleanedChildrens) { if (relType.equals("isRelatedTo")) { - list.addAll(enrichMisissingPublicationIsRelatedTo.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMisissingPublicationIsRelatedTo + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } else if (relType.equals("references")) { - list.addAll(enrichMissingPublicationReferences.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMissingPublicationReferences + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } else if (relType.equals("isReferencedBy")) { - list.addAll(enrichMissingPublicationIsReferencedBy.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMissingPublicationIsReferencedBy + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } else if (relType.equals("isSupplementedTo")) { - list.addAll(enrichMissingPublicationIsSupplementedTo.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMissingPublicationIsSupplementedTo + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } else if (relType.equals("isSupplementedBy")) { - list.addAll(enrichMissingPublicationIsSupplementedBy.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMissingPublicationIsSupplementedBy + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } } @@ -310,15 +330,29 @@ public class GenerateEventsApplication { for (final Pair> target : cleanedChildrens) { if (relType.equals("isRelatedTo")) { - list.addAll(enrichMisissingDatasetIsRelatedTo.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMisissingDatasetIsRelatedTo + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } else if (relType.equals("references")) { - list.addAll(enrichMissingDatasetReferences.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMissingDatasetReferences.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } else if (relType.equals("isReferencedBy")) { - list.addAll(enrichMissingDatasetIsReferencedBy.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMissingDatasetIsReferencedBy + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } else if (relType.equals("isSupplementedTo")) { - list.addAll(enrichMissingDatasetIsSupplementedTo.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMissingDatasetIsSupplementedTo + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } else if (relType.equals("isSupplementedBy")) { - list.addAll(enrichMissingDatasetIsSupplementedBy.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); + list + .addAll( + enrichMissingDatasetIsSupplementedBy + .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); } } @@ -339,8 +373,12 @@ public class GenerateEventsApplication { private static DedupConfig loadDedupConfig(final String isLookupUrl, final String profId) throws Exception { final ISLookUpService isLookUpService = ISLookupClientFactory.getLookUpService(isLookupUrl); - final String conf = isLookUpService.getResourceProfileByQuery(String - .format("for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()", profId)); + final String conf = isLookUpService + .getResourceProfileByQuery( + String + .format( + "for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()", + profId)); final DedupConfig dedupConfig = new ObjectMapper().readValue(conf, DedupConfig.class); dedupConfig.getPace().initModel(); diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java index 286b40ad5..95d43ae68 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java @@ -22,7 +22,8 @@ public abstract class UpdateMatcher { this.multipleUpdate = multipleUpdate; } - public Collection> searchUpdatesForRecord(final K res, final Collection others, final DedupConfig dedupConfig) { + public Collection> searchUpdatesForRecord(final K res, final Collection others, + final DedupConfig dedupConfig) { final Map> infoMap = new HashMap<>(); @@ -30,7 +31,8 @@ public abstract class UpdateMatcher { if (source != res) { for (final UpdateInfo info : findUpdates(source, res, dedupConfig)) { final String s = DigestUtils.md5Hex(info.getHighlightValueAsString()); - if (!infoMap.containsKey(s) || infoMap.get(s).getTrust() < info.getTrust()) {} else { + if (!infoMap.containsKey(s) || infoMap.get(s).getTrust() < info.getTrust()) { + } else { infoMap.put(s, info); } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java index c3b6bda66..7dc340b3c 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java @@ -18,9 +18,11 @@ public class EnrichMissingAbstract extends UpdateMatcher { } @Override - protected List> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { if (isMissing(target.getDescription()) && !isMissing(source.getDescription())) { - return Arrays.asList(generateUpdateInfo(source.getDescription().get(0).getValue(), source, target, dedupConfig)); + return Arrays + .asList(generateUpdateInfo(source.getDescription().get(0).getValue(), source, target, dedupConfig)); } return new ArrayList<>(); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java index 89292d3da..7a1677ae2 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java @@ -19,7 +19,8 @@ public class EnrichMissingAuthorOrcid extends UpdateMatcher>> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List>> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { // TODO // return Arrays.asList(new EnrichMissingAbstract("xxxxxxx", 0.9f)); return Arrays.asList(); diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java index 7f5a595cc..d14490ba8 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java @@ -21,7 +21,8 @@ public class EnrichMissingOpenAccess extends UpdateMatcher { } @Override - protected List> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { final long count = target .getInstance() .stream() @@ -29,7 +30,9 @@ public class EnrichMissingOpenAccess extends UpdateMatcher { .filter(right -> right.equals(BrokerConstants.OPEN_ACCESS)) .count(); - if (count > 0) { return Arrays.asList(); } + if (count > 0) { + return Arrays.asList(); + } return source .getInstance() diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java index 6e106e669..20303ec1b 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java @@ -20,10 +20,13 @@ public class EnrichMissingPid extends UpdateMatcher { } @Override - protected List> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { final long count = target.getPid().size(); - if (count > 0) { return Arrays.asList(); } + if (count > 0) { + return Arrays.asList(); + } return source .getPid() @@ -33,7 +36,8 @@ public class EnrichMissingPid extends UpdateMatcher { .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo(final Pid highlightValue, final Result source, final Result target, final DedupConfig dedupConfig) { + public UpdateInfo generateUpdateInfo(final Pid highlightValue, final Result source, final Result target, + final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_PID, highlightValue, source, target, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java index d2b28d65d..e1de8ce4d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java @@ -18,9 +18,11 @@ public class EnrichMissingPublicationDate extends UpdateMatcher } @Override - protected List> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { if (isMissing(target.getDateofacceptance()) && !isMissing(source.getDateofacceptance())) { - return Arrays.asList(generateUpdateInfo(source.getDateofacceptance().getValue(), source, target, dedupConfig)); + return Arrays + .asList(generateUpdateInfo(source.getDateofacceptance().getValue(), source, target, dedupConfig)); } return new ArrayList<>(); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java index de888ff87..c51f8991c 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java @@ -23,7 +23,8 @@ public class EnrichMissingSubject extends UpdateMatcher>> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List>> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { final Set existingTypes = target .getSubject() .stream() diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java index 021449797..2ac04fd12 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java @@ -21,7 +21,8 @@ public class EnrichMoreOpenAccess extends UpdateMatcher { } @Override - protected List> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { final Set urls = target .getInstance() .stream() diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java index c64ed20ea..e4bf5d2c2 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java @@ -20,7 +20,8 @@ public class EnrichMorePid extends UpdateMatcher { } @Override - protected List> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { final Set existingPids = target .getPid() .stream() @@ -36,7 +37,8 @@ public class EnrichMorePid extends UpdateMatcher { .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo(final Pid highlightValue, final Result source, final Result target, final DedupConfig dedupConfig) { + public UpdateInfo generateUpdateInfo(final Pid highlightValue, final Result source, final Result target, + final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MORE_PID, highlightValue, source, target, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java index 3f7f5b3d5..d6e607c31 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java @@ -21,7 +21,8 @@ public class EnrichMoreSubject extends UpdateMatcher>> findUpdates(final Result source, final Result target, final DedupConfig dedupConfig) { + protected List>> findUpdates(final Result source, final Result target, + final DedupConfig dedupConfig) { final Set existingSubjects = target .getSubject() .stream() diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventGroup.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventGroup.java index 9c7081c79..25c7698a0 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventGroup.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventGroup.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.broker.oa.util; import java.io.Serializable; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultAggregator.java index 94685eeae..475c76814 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultAggregator.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultAggregator.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.broker.oa.util; import org.apache.spark.sql.Encoder; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultGroup.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultGroup.java index 8fe7a5939..2be673db0 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultGroup.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultGroup.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.broker.oa.util; import java.io.Serializable; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java index 6bf59c125..5338d4f3d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java @@ -1,14 +1,22 @@ + package eu.dnetlib.dhp.broker.oa.util; public class TrustUtils { public static float rescale(final double score, final double threshold) { - if (score >= BrokerConstants.MAX_TRUST) { return BrokerConstants.MAX_TRUST; } + if (score >= BrokerConstants.MAX_TRUST) { + return BrokerConstants.MAX_TRUST; + } - final double val = (score - threshold) * (BrokerConstants.MAX_TRUST - BrokerConstants.MIN_TRUST) / (BrokerConstants.MAX_TRUST - threshold); + final double val = (score - threshold) * (BrokerConstants.MAX_TRUST - BrokerConstants.MIN_TRUST) + / (BrokerConstants.MAX_TRUST - threshold); - if (val < BrokerConstants.MIN_TRUST) { return BrokerConstants.MIN_TRUST; } - if (val > BrokerConstants.MAX_TRUST) { return BrokerConstants.MAX_TRUST; } + if (val < BrokerConstants.MIN_TRUST) { + return BrokerConstants.MIN_TRUST; + } + if (val > BrokerConstants.MAX_TRUST) { + return BrokerConstants.MAX_TRUST; + } return (float) val; } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java index de6a71397..893aa2827 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java @@ -68,8 +68,10 @@ public final class UpdateInfo { private float calculateTrust(final DedupConfig dedupConfig, final Result r1, final Result r2) { try { final ObjectMapper objectMapper = new ObjectMapper(); - final MapDocument doc1 = MapDocumentUtil.asMapDocumentWithJPath(dedupConfig, objectMapper.writeValueAsString(r1)); - final MapDocument doc2 = MapDocumentUtil.asMapDocumentWithJPath(dedupConfig, objectMapper.writeValueAsString(r2)); + final MapDocument doc1 = MapDocumentUtil + .asMapDocumentWithJPath(dedupConfig, objectMapper.writeValueAsString(r1)); + final MapDocument doc2 = MapDocumentUtil + .asMapDocumentWithJPath(dedupConfig, objectMapper.writeValueAsString(r2)); final double score = new TreeProcessor(dedupConfig).computeScore(doc1, doc2); final double threshold = dedupConfig.getWf().getThreshold(); @@ -118,7 +120,8 @@ public final class UpdateInfo { .map(Instance::getUrl) .flatMap(List::stream) .findFirst() - .orElse(null);; + .orElse(null); + ; final Provenance provenance = new Provenance().setId(provId).setRepositoryName(provRepo).setUrl(provUrl); diff --git a/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/TrustUtilsTest.java b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/TrustUtilsTest.java index 58f391c24..bb23d6085 100644 --- a/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/TrustUtilsTest.java +++ b/dhp-workflows/dhp-broker-events/src/test/java/eu/dnetlib/dhp/broker/oa/util/TrustUtilsTest.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.broker.oa.util; import static org.junit.jupiter.api.Assertions.assertTrue; From c77fc684849f8a2bcc7423936983599c2b6e5516 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 10 Jun 2020 14:49:25 +0200 Subject: [PATCH 55/88] restored Saxon-HE dependency definition --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index f4b96fefb..e0ee18900 100644 --- a/pom.xml +++ b/pom.xml @@ -193,7 +193,6 @@ net.sf.saxon Saxon-HE 9.9.1-6 - provided From f1bce643910514f956fff7180a04631e7adf21e5 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 10 Jun 2020 21:36:31 +0200 Subject: [PATCH 56/88] WIP: graph cleaner implementation --- .../oa/graph/clean/NormalizeEmptyFields.java | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java new file mode 100644 index 000000000..77537801f --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java @@ -0,0 +1,88 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils; +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.oaf.Oaf; +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.api.java.function.MapFunction; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; + +public class NormalizeEmptyFields implements MapFunction { + + private VocabularyGroup vocabularies; + + public NormalizeEmptyFields(VocabularyGroup vocabularies) { + this.vocabularies = vocabularies; + } + + @Override + public T call(T value) throws Exception { + + doNormalize(value); + + return value; + } + + private void doNormalize(Object o) { + if (Objects.isNull(o)) { + return; + } + + if (o instanceof Iterable) { + for (Object oi : (Iterable) o) { + doNormalize(oi); + } + } else { + + Class clazz = o.getClass(); + + if (clazz.isPrimitive() + || o instanceof Integer + || o instanceof Double + || o instanceof Float + || o instanceof Long + || o instanceof Boolean + || o instanceof String) { + return; + } else { + try { + for (Field field : getAllFields(new LinkedList<>(), clazz)) { + field.setAccessible(true); + Object value = field.get(o); + if (value instanceof Qualifier && Objects.isNull(value)) { + field.set(o, OafMapperUtils.unknown("", "")); + } else if (value instanceof Field && Objects.isNull(value)) { + + } else { + doNormalize(value); + } + } + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new RuntimeException(e); + } + } + } + } + + private static List getAllFields(List fields, Class clazz) { + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + + final Class superclass = clazz.getSuperclass(); + if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { + getAllFields(fields, superclass); + } + + return fields; + } + + public VocabularyGroup getVocabularies() { + return vocabularies; + } +} From d1d92c4d8c07caa363d9c8e6e3176a0ac0cd9ed4 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 11 Jun 2020 10:12:00 +0200 Subject: [PATCH 57/88] fixed integration of claims in the graph --- .../main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java | 4 ++++ .../dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java new file mode 100644 index 000000000..c329a3111 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java @@ -0,0 +1,4 @@ +package eu.dnetlib.dhp.oa.graph.clean; + +public class OafNavigator { +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java index f52587d73..3568dc52a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java @@ -141,8 +141,10 @@ public class GenerateEntitiesApplication { switch (type.toLowerCase()) { case "oaf-store-cleaned": + case "oaf-store-claim": return new OafToOafMapper(vocs, false).processMdRecord(s); case "odf-store-cleaned": + case "odf-store-claim": return new OdfToOafMapper(vocs, false).processMdRecord(s); case "oaf-store-intersection": return new OafToOafMapper(vocs, true).processMdRecord(s); From 99f88e1cb8298efec11487f646d4e3317b8c453c Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 11 Jun 2020 10:51:57 +0200 Subject: [PATCH 58/88] fixed generation entities from claims --- .../dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java index 4262732a5..d32117063 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java @@ -137,8 +137,10 @@ public class GenerateEntitiesApplication { final String type = StringUtils.substringAfter(id, ":"); switch (type.toLowerCase()) { + case "oaf-store-claim": case "oaf-store-cleaned": return new OafToOafMapper(vocs, false).processMdRecord(s); + case "odf-store-claim": case "odf-store-cleaned": return new OdfToOafMapper(vocs, false).processMdRecord(s); case "oaf-store-intersection": From 04fdcacd837957a44813c592ba0bf528b68d7946 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 11 Jun 2020 11:25:18 +0200 Subject: [PATCH 59/88] results with all joined entities --- .../dhp/broker/model/EventFactory.java | 6 +- .../broker/oa/GenerateEventsApplication.java | 303 ++++-------------- .../dhp/broker/oa/matchers/UpdateMatcher.java | 11 +- .../AbstractEnrichMissingDataset.java | 28 +- .../EnrichMissingDatasetIsReferencedBy.java | 5 + .../EnrichMissingDatasetIsRelatedTo.java | 5 + .../EnrichMissingDatasetIsSupplementedBy.java | 5 + .../EnrichMissingDatasetIsSupplementedTo.java | 5 + .../EnrichMissingDatasetReferences.java | 5 + .../relatedProjects/EnrichMissingProject.java | 23 +- .../relatedProjects/EnrichMoreProject.java | 23 +- .../AbstractEnrichMissingPublication.java | 27 +- ...nrichMissingPublicationIsReferencedBy.java | 4 + .../EnrichMissingPublicationIsRelatedTo.java | 5 + ...ichMissingPublicationIsSupplementedBy.java | 4 + ...ichMissingPublicationIsSupplementedTo.java | 5 + .../EnrichMissingPublicationReferences.java | 5 + .../EnrichMissingSoftware.java | 23 +- .../relatedSoftware/EnrichMoreSoftware.java | 23 +- .../simple/EnrichMissingAbstract.java | 17 +- .../simple/EnrichMissingAuthorOrcid.java | 11 +- .../simple/EnrichMissingOpenAccess.java | 13 +- .../oa/matchers/simple/EnrichMissingPid.java | 14 +- .../simple/EnrichMissingPublicationDate.java | 18 +- .../matchers/simple/EnrichMissingSubject.java | 13 +- .../matchers/simple/EnrichMoreOpenAccess.java | 13 +- .../oa/matchers/simple/EnrichMorePid.java | 13 +- .../oa/matchers/simple/EnrichMoreSubject.java | 13 +- .../dhp/broker/oa/util/EventFinder.java | 86 +++++ .../dhp/broker/oa/util/UpdateInfo.java | 20 +- .../simple}/ResultAggregator.java | 8 +- .../{ => aggregators/simple}/ResultGroup.java | 10 +- .../aggregators/withRels/RelatedDataset.java | 30 ++ .../withRels/RelatedEntityFactory.java | 28 ++ .../aggregators/withRels/RelatedProject.java | 30 ++ .../withRels/RelatedPublication.java | 30 ++ .../aggregators/withRels/RelatedSoftware.java | 30 ++ .../withRels/ResultWithRelations.java | 55 ++++ .../ResultWithRelationsAggregator.java | 68 ++++ 39 files changed, 650 insertions(+), 385 deletions(-) create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java rename dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/{ => aggregators/simple}/ResultAggregator.java (75%) rename dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/{ => aggregators/simple}/ResultGroup.java (58%) create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelations.java create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelationsAggregator.java diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java index df33fae0d..9146cf422 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java @@ -37,7 +37,7 @@ public class EventFactory { final Map map = createMapFromResult(updateInfo); final String eventId = calculateEventId( - updateInfo.getTopicPath(), updateInfo.getTarget().getOriginalId().get(0), + updateInfo.getTopicPath(), updateInfo.getTarget().getResult().getOriginalId().get(0), updateInfo.getHighlightValueAsString()); res.setEventId(eventId); @@ -54,8 +54,8 @@ public class EventFactory { private static Map createMapFromResult(final UpdateInfo updateInfo) { final Map map = new HashMap<>(); - final Result source = updateInfo.getSource(); - final Result target = updateInfo.getTarget(); + final Result source = updateInfo.getSource().getResult(); + final Result target = updateInfo.getTarget().getResult(); final List collectedFrom = target.getCollectedfrom(); if (collectedFrom.size() == 1) { diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java index ecf4e3eff..a09767192 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java @@ -3,15 +3,9 @@ package eu.dnetlib.dhp.broker.oa; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; @@ -26,38 +20,16 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.broker.model.Event; -import eu.dnetlib.dhp.broker.model.EventFactory; -import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsReferencedBy; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsRelatedTo; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsSupplementedBy; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsSupplementedTo; -import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetReferences; -import eu.dnetlib.dhp.broker.oa.matchers.relatedProjects.EnrichMissingProject; -import eu.dnetlib.dhp.broker.oa.matchers.relatedProjects.EnrichMoreProject; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsReferencedBy; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsRelatedTo; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsSupplementedBy; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsSupplementedTo; -import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationReferences; -import eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware.EnrichMissingSoftware; -import eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware.EnrichMoreSoftware; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingAbstract; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingAuthorOrcid; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingOpenAccess; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPid; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPublicationDate; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingSubject; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreOpenAccess; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMorePid; -import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreSubject; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import eu.dnetlib.dhp.broker.oa.util.EventFinder; import eu.dnetlib.dhp.broker.oa.util.EventGroup; -import eu.dnetlib.dhp.broker.oa.util.ResultAggregator; -import eu.dnetlib.dhp.broker.oa.util.ResultGroup; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultAggregator; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedEntityFactory; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelationsAggregator; import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.schema.oaf.OafEntity; import eu.dnetlib.dhp.schema.oaf.Project; import eu.dnetlib.dhp.schema.oaf.Publication; import eu.dnetlib.dhp.schema.oaf.Relation; @@ -72,40 +44,6 @@ public class GenerateEventsApplication { private static final Logger log = LoggerFactory.getLogger(GenerateEventsApplication.class); - // Simple Matchers - private static final UpdateMatcher enrichMissingAbstract = new EnrichMissingAbstract(); - private static final UpdateMatcher enrichMissingAuthorOrcid = new EnrichMissingAuthorOrcid(); - private static final UpdateMatcher enrichMissingOpenAccess = new EnrichMissingOpenAccess(); - private static final UpdateMatcher enrichMissingPid = new EnrichMissingPid(); - private static final UpdateMatcher enrichMissingPublicationDate = new EnrichMissingPublicationDate(); - private static final UpdateMatcher enrichMissingSubject = new EnrichMissingSubject(); - private static final UpdateMatcher enrichMoreOpenAccess = new EnrichMoreOpenAccess(); - private static final UpdateMatcher enrichMorePid = new EnrichMorePid(); - private static final UpdateMatcher enrichMoreSubject = new EnrichMoreSubject(); - - // Advanced matchers - private static final UpdateMatcher>, ?> enrichMissingProject = new EnrichMissingProject(); - private static final UpdateMatcher>, ?> enrichMoreProject = new EnrichMoreProject(); - - private static final UpdateMatcher>, ?> enrichMissingSoftware = new EnrichMissingSoftware(); - private static final UpdateMatcher>, ?> enrichMoreSoftware = new EnrichMoreSoftware(); - - private static final UpdateMatcher>, ?> enrichMisissingPublicationIsRelatedTo = new EnrichMissingPublicationIsRelatedTo(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsReferencedBy = new EnrichMissingPublicationIsReferencedBy(); - private static final UpdateMatcher>, ?> enrichMissingPublicationReferences = new EnrichMissingPublicationReferences(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsSupplementedTo = new EnrichMissingPublicationIsSupplementedTo(); - private static final UpdateMatcher>, ?> enrichMissingPublicationIsSupplementedBy = new EnrichMissingPublicationIsSupplementedBy(); - - private static final UpdateMatcher>, ?> enrichMisissingDatasetIsRelatedTo = new EnrichMissingDatasetIsRelatedTo(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsReferencedBy = new EnrichMissingDatasetIsReferencedBy(); - private static final UpdateMatcher>, ?> enrichMissingDatasetReferences = new EnrichMissingDatasetReferences(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsSupplementedTo = new EnrichMissingDatasetIsSupplementedTo(); - private static final UpdateMatcher>, ?> enrichMissingDatasetIsSupplementedBy = new EnrichMissingDatasetIsSupplementedBy(); - - // Aggregators - private static final TypedColumn, ResultGroup> resultAggrTypedColumn = new ResultAggregator() - .toColumn(); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static void main(final String[] args) throws Exception { @@ -145,14 +83,10 @@ public class GenerateEventsApplication { final Dataset all = spark.emptyDataset(Encoders.kryo(Event.class)); for (final Class r1 : BrokerConstants.RESULT_CLASSES) { - all.union(generateSimpleEvents(spark, graphPath, r1, dedupConfig)); - - for (final Class r2 : BrokerConstants.RESULT_CLASSES) { - all.union(generateRelationEvents(spark, graphPath, r1, r2, dedupConfig)); - } + all.union(generateEvents(spark, graphPath, r1, dedupConfig)); } - all.write().mode(SaveMode.Overwrite).json(eventsPath); + all.write().mode(SaveMode.Overwrite).option("compression", "gzip").json(eventsPath); }); } @@ -161,203 +95,83 @@ public class GenerateEventsApplication { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } - private static Dataset generateSimpleEvents(final SparkSession spark, + private static Dataset generateEvents( + final SparkSession spark, final String graphPath, - final Class resultClazz, + final Class sourceClass, final DedupConfig dedupConfig) { - final Dataset results = readPath( - spark, graphPath + "/" + resultClazz.getSimpleName().toLowerCase(), Result.class) - .filter(r -> r.getDataInfo().getDeletedbyinference()); + final Dataset results = expandResultsWithRelations(spark, graphPath, sourceClass); final Dataset mergedRels = readPath(spark, graphPath + "/relation", Relation.class) .filter(r -> r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); + final TypedColumn, ResultGroup> aggr = new ResultAggregator() + .toColumn(); + return results - .joinWith(mergedRels, results.col("id").equalTo(mergedRels.col("source")), "inner") - .groupByKey((MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) - .agg(resultAggrTypedColumn) + .joinWith(mergedRels, results.col("result.id").equalTo(mergedRels.col("source")), "inner") + .groupByKey( + (MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) + .agg(aggr) .map((MapFunction, ResultGroup>) t -> t._2, Encoders.kryo(ResultGroup.class)) .filter(ResultGroup::isValid) .map( - (MapFunction) g -> GenerateEventsApplication - .generateSimpleEvents(g, dedupConfig), + (MapFunction) g -> EventFinder.generateEvents(g, dedupConfig), Encoders.kryo(EventGroup.class)) .flatMap(group -> group.getData().iterator(), Encoders.kryo(Event.class)); } - private static EventGroup generateSimpleEvents(final ResultGroup results, final DedupConfig dedupConfig) { - final List> list = new ArrayList<>(); - - for (final Result target : results.getData()) { - list.addAll(enrichMissingAbstract.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - list.addAll(enrichMissingAuthorOrcid.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - list.addAll(enrichMissingOpenAccess.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - list.addAll(enrichMissingPid.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - list.addAll(enrichMissingPublicationDate.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - list.addAll(enrichMissingSubject.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - list.addAll(enrichMoreOpenAccess.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - list.addAll(enrichMorePid.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - list.addAll(enrichMoreSubject.searchUpdatesForRecord(target, results.getData(), dedupConfig)); - } - - final EventGroup events = new EventGroup(); - list.stream().map(EventFactory::newBrokerEvent).forEach(events::addElement); - return events; - } - - private static Dataset generateRelationEvents( + private static Dataset expandResultsWithRelations( final SparkSession spark, final String graphPath, - final Class sourceClass, - final Class targetClass, - final DedupConfig dedupConfig) { - - final Dataset sources = readPath( - spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) - .filter(r -> r.getDataInfo().getDeletedbyinference()); - - final Dataset targets = readPath( - spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), targetClass); - - final Dataset mergedRels = readPath(spark, graphPath + "/relation", Relation.class) - .filter(r -> r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); + final Class sourceClass) { + final Dataset projects = readPath(spark, graphPath + "/project", Project.class); + final Dataset datasets = readPath( + spark, graphPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class); + final Dataset softwares = readPath(spark, graphPath + "/software", Software.class); + final Dataset publications = readPath(spark, graphPath + "/publication", Publication.class); final Dataset rels = readPath(spark, graphPath + "/relation", Relation.class) .filter(r -> !r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); - final Dataset duplicates = sources - .joinWith(mergedRels, sources.col("id").equalTo(rels.col("source")), "inner") - .groupByKey((MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) - .agg(resultAggrTypedColumn) - .map((MapFunction, ResultGroup>) t -> t._2, Encoders.kryo(ResultGroup.class)) - .filter(ResultGroup::isValid); + final Dataset r0 = readPath( + spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()) + .map(r -> new ResultWithRelations(r), Encoders.kryo(ResultWithRelations.class)); + final Dataset r1 = join(r0, rels, relatedEntities(projects, rels, RelatedProject.class)); + final Dataset r2 = join(r1, rels, relatedEntities(softwares, rels, RelatedProject.class)); + final Dataset r3 = join(r2, rels, relatedEntities(datasets, rels, RelatedProject.class)); + final Dataset r4 = join( + r3, rels, relatedEntities(publications, rels, RelatedProject.class)); + ; - if (targetClass == Project.class) { - // TODO join using: generateProjectsEvents - } else if (targetClass == Software.class) { - // TODO join using: generateSoftwareEvents - } else if (targetClass == Publication.class) { - // TODO join using: generatePublicationRelatedEvents - } else if (targetClass == eu.dnetlib.dhp.schema.oaf.Dataset.class) { - // TODO join using: generateDatasetRelatedEvents - } - - return null; + return r4; } - private List generateProjectsEvents(final Collection>> childrenWithProjects, - final DedupConfig dedupConfig) { - final List> list = new ArrayList<>(); - - for (final Pair> target : childrenWithProjects) { - list.addAll(enrichMissingProject.searchUpdatesForRecord(target, childrenWithProjects, dedupConfig)); - list.addAll(enrichMoreProject.searchUpdatesForRecord(target, childrenWithProjects, dedupConfig)); - } - - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); + private static Dataset relatedEntities(final Dataset targets, final Dataset rels, + final Class clazz) { + return rels + .joinWith(targets, targets.col("id").equalTo(rels.col("target")), "inner") + .map( + t -> RelatedEntityFactory.newRelatedEntity(t._1.getSource(), t._1.getRelType(), t._2, clazz), + Encoders.kryo(clazz)); } - private List generateSoftwareEvents(final Collection>> childrenWithSoftwares, - final DedupConfig dedupConfig) { - final List> list = new ArrayList<>(); + private static Dataset join(final Dataset sources, + final Dataset rels, + final Dataset typedRels) { - for (final Pair> target : childrenWithSoftwares) { - list.addAll(enrichMissingSoftware.searchUpdatesForRecord(target, childrenWithSoftwares, dedupConfig)); - list.addAll(enrichMoreSoftware.searchUpdatesForRecord(target, childrenWithSoftwares, dedupConfig)); - } - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); - } - - private List generatePublicationRelatedEvents(final String relType, - final Collection>>> childrenWithRels, - final DedupConfig dedupConfig) { - - final List> list = new ArrayList<>(); - - final List>> cleanedChildrens = childrenWithRels - .stream() - .filter(p -> p.getRight().containsKey(relType)) - .map(p -> Pair.of(p.getLeft(), p.getRight().get(relType))) - .filter(p -> p.getRight().size() > 0) - .collect(Collectors.toList()); - - for (final Pair> target : cleanedChildrens) { - if (relType.equals("isRelatedTo")) { - list - .addAll( - enrichMisissingPublicationIsRelatedTo - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } else if (relType.equals("references")) { - list - .addAll( - enrichMissingPublicationReferences - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } else if (relType.equals("isReferencedBy")) { - list - .addAll( - enrichMissingPublicationIsReferencedBy - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } else if (relType.equals("isSupplementedTo")) { - list - .addAll( - enrichMissingPublicationIsSupplementedTo - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } else if (relType.equals("isSupplementedBy")) { - list - .addAll( - enrichMissingPublicationIsSupplementedBy - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } - } - - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); - - } - - private List generateDatasetRelatedEvents(final String relType, - final Collection>>> childrenWithRels, - final DedupConfig dedupConfig) { - - final List> list = new ArrayList<>(); - - final List>> cleanedChildrens = childrenWithRels - .stream() - .filter(p -> p.getRight().containsKey(relType)) - .map(p -> Pair.of(p.getLeft(), p.getRight().get(relType))) - .filter(p -> p.getRight().size() > 0) - .collect(Collectors.toList()); - - for (final Pair> target : cleanedChildrens) { - if (relType.equals("isRelatedTo")) { - list - .addAll( - enrichMisissingDatasetIsRelatedTo - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } else if (relType.equals("references")) { - list - .addAll( - enrichMissingDatasetReferences.searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } else if (relType.equals("isReferencedBy")) { - list - .addAll( - enrichMissingDatasetIsReferencedBy - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } else if (relType.equals("isSupplementedTo")) { - list - .addAll( - enrichMissingDatasetIsSupplementedTo - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } else if (relType.equals("isSupplementedBy")) { - list - .addAll( - enrichMissingDatasetIsSupplementedBy - .searchUpdatesForRecord(target, cleanedChildrens, dedupConfig)); - } - } - - return list.stream().map(EventFactory::newBrokerEvent).collect(Collectors.toList()); + final TypedColumn, ResultWithRelations> aggr = new ResultWithRelationsAggregator() + .toColumn(); + ; + return sources + .joinWith(typedRels, sources.col("result.id").equalTo(rels.col("source")), "left_outer") + .groupByKey( + (MapFunction, String>) t -> t._1.getResult().getId(), Encoders.STRING()) + .agg(aggr) + .map(t -> t._2, Encoders.kryo(ResultWithRelations.class)); } public static Dataset readPath( @@ -386,7 +200,6 @@ public class GenerateEventsApplication { // dedupConfig.getWf().setConfigurationId("???"); return dedupConfig; - } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java index 95d43ae68..fd87d81dd 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java @@ -11,10 +11,11 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Field; import eu.dnetlib.pace.config.DedupConfig; -public abstract class UpdateMatcher { +public abstract class UpdateMatcher { private final boolean multipleUpdate; @@ -22,12 +23,13 @@ public abstract class UpdateMatcher { this.multipleUpdate = multipleUpdate; } - public Collection> searchUpdatesForRecord(final K res, final Collection others, + public Collection> searchUpdatesForRecord(final ResultWithRelations res, + final Collection others, final DedupConfig dedupConfig) { final Map> infoMap = new HashMap<>(); - for (final K source : others) { + for (final ResultWithRelations source : others) { if (source != res) { for (final UpdateInfo info : findUpdates(source, res, dedupConfig)) { final String s = DigestUtils.md5Hex(info.getHighlightValueAsString()); @@ -53,7 +55,8 @@ public abstract class UpdateMatcher { } } - protected abstract List> findUpdates(K source, K target, DedupConfig dedupConfig); + protected abstract List> findUpdates(ResultWithRelations source, ResultWithRelations target, + DedupConfig dedupConfig); protected static boolean isMissing(final List> list) { return list == null || list.isEmpty() || StringUtils.isBlank(list.get(0).getValue()); diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java index 3cf7b18f9..a2ce32a9d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java @@ -5,18 +5,17 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedDataset; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Dataset; -import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.pace.config.DedupConfig; public abstract class AbstractEnrichMissingDataset - extends UpdateMatcher>, eu.dnetlib.broker.objects.Dataset> { + extends UpdateMatcher { private final Topic topic; @@ -25,21 +24,27 @@ public abstract class AbstractEnrichMissingDataset this.topic = topic; } + protected abstract boolean filterByType(String relType); + @Override protected final List> findUpdates( - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final Set existingDatasets = target - .getRight() + .getDatasets() .stream() + .filter(rel -> filterByType(rel.getRelType())) + .map(RelatedDataset::getRelDataset) .map(Dataset::getId) .collect(Collectors.toSet()); return source - .getRight() + .getDatasets() .stream() + .filter(rel -> filterByType(rel.getRelType())) + .map(RelatedDataset::getRelDataset) .filter(d -> !existingDatasets.contains(d.getId())) .map(ConversionUtils::oafDatasetToBrokerDataset) .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) @@ -49,12 +54,12 @@ public abstract class AbstractEnrichMissingDataset protected final UpdateInfo generateUpdateInfo( final eu.dnetlib.broker.objects.Dataset highlightValue, - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( getTopic(), - highlightValue, source.getLeft(), target.getLeft(), + highlightValue, source, target, (p, rel) -> p.getDatasets().add(rel), rel -> rel.getInstances().get(0).getUrl(), dedupConfig); @@ -63,4 +68,5 @@ public abstract class AbstractEnrichMissingDataset public Topic getTopic() { return topic; } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsReferencedBy.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsReferencedBy.java index 74ce761f4..21786687e 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsReferencedBy.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsReferencedBy.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetIsReferencedBy extends AbstractEnrichMissingDat super(Topic.ENRICH_MISSING_DATASET_IS_REFERENCED_BY); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isReferencedBy"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsRelatedTo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsRelatedTo.java index 05a891059..0f3739434 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsRelatedTo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsRelatedTo.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetIsRelatedTo extends AbstractEnrichMissingDatase super(Topic.ENRICH_MISSING_DATASET_IS_RELATED_TO); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isRelatedTo"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedBy.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedBy.java index 23bd68fa1..cde227fee 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedBy.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedBy.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetIsSupplementedBy extends AbstractEnrichMissingD super(Topic.ENRICH_MISSING_DATASET_IS_SUPPLEMENTED_BY); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isSupplementedBy"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedTo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedTo.java index 03160b6f0..750165ff5 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedTo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetIsSupplementedTo.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetIsSupplementedTo extends AbstractEnrichMissingD super(Topic.ENRICH_MISSING_DATASET_IS_SUPPLEMENTED_TO); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isSupplementedTo"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetReferences.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetReferences.java index bf1df053d..b1c0afe16 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetReferences.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/EnrichMissingDatasetReferences.java @@ -9,4 +9,9 @@ public class EnrichMissingDatasetReferences extends AbstractEnrichMissingDataset super(Topic.ENRICH_MISSING_DATASET_REFERENCES); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("references"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java index 22817a25d..546287795 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java @@ -5,34 +5,33 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Project; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingProject - extends UpdateMatcher>, eu.dnetlib.broker.objects.Project> { + extends UpdateMatcher { public EnrichMissingProject() { super(true); } @Override - protected List> findUpdates(final Pair> source, - final Pair> target, + protected List> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { - if (source.getRight().isEmpty()) { + if (source.getProjects().isEmpty()) { return Arrays.asList(); } else { return target - .getRight() + .getProjects() .stream() + .map(RelatedProject::getRelProject) .map(ConversionUtils::oafProjectToBrokerProject) .map(p -> generateUpdateInfo(p, source, target, dedupConfig)) .collect(Collectors.toList()); @@ -41,12 +40,12 @@ public class EnrichMissingProject public UpdateInfo generateUpdateInfo( final eu.dnetlib.broker.objects.Project highlightValue, - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_PROJECT, - highlightValue, source.getLeft(), target.getLeft(), + highlightValue, source, target, (p, prj) -> p.getProjects().add(prj), prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode(), dedupConfig); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java index 016bdd283..54ebe7b71 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java @@ -5,36 +5,37 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Project; -import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMoreProject extends UpdateMatcher>, eu.dnetlib.broker.objects.Project> { +public class EnrichMoreProject extends UpdateMatcher { public EnrichMoreProject() { super(true); } @Override - protected List> findUpdates(final Pair> source, - final Pair> target, + protected List> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final Set existingProjects = source - .getRight() + .getProjects() .stream() + .map(RelatedProject::getRelProject) .map(Project::getId) .collect(Collectors.toSet()); return target - .getRight() + .getProjects() .stream() + .map(RelatedProject::getRelProject) .filter(p -> !existingProjects.contains(p.getId())) .map(ConversionUtils::oafProjectToBrokerProject) .map(p -> generateUpdateInfo(p, source, target, dedupConfig)) @@ -43,12 +44,12 @@ public class EnrichMoreProject extends UpdateMatcher> public UpdateInfo generateUpdateInfo( final eu.dnetlib.broker.objects.Project highlightValue, - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MORE_PROJECT, - highlightValue, source.getLeft(), target.getLeft(), + highlightValue, source, target, (p, prj) -> p.getProjects().add(prj), prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode(), dedupConfig); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java index ec575e68d..8793d38dc 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java @@ -5,18 +5,17 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedPublication; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.pace.config.DedupConfig; public abstract class AbstractEnrichMissingPublication - extends UpdateMatcher>, eu.dnetlib.broker.objects.Publication> { + extends UpdateMatcher { private final Topic topic; @@ -25,21 +24,27 @@ public abstract class AbstractEnrichMissingPublication this.topic = topic; } + protected abstract boolean filterByType(String relType); + @Override protected final List> findUpdates( - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final Set existingPublications = target - .getRight() + .getPublications() .stream() + .filter(rel -> filterByType(rel.getRelType())) + .map(RelatedPublication::getRelPublication) .map(Publication::getId) .collect(Collectors.toSet()); return source - .getRight() + .getPublications() .stream() + .filter(rel -> filterByType(rel.getRelType())) + .map(RelatedPublication::getRelPublication) .filter(d -> !existingPublications.contains(d.getId())) .map(ConversionUtils::oafResultToBrokerPublication) .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) @@ -49,12 +54,12 @@ public abstract class AbstractEnrichMissingPublication protected final UpdateInfo generateUpdateInfo( final eu.dnetlib.broker.objects.Publication highlightValue, - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( getTopic(), - highlightValue, source.getLeft(), target.getLeft(), + highlightValue, source, target, (p, rel) -> p.getPublications().add(rel), rel -> rel.getInstances().get(0).getUrl(), dedupConfig); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsReferencedBy.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsReferencedBy.java index 73fa8a45f..eebb5c1a6 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsReferencedBy.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsReferencedBy.java @@ -9,4 +9,8 @@ public class EnrichMissingPublicationIsReferencedBy extends AbstractEnrichMissin super(Topic.ENRICH_MISSING_PUBLICATION_IS_REFERENCED_BY); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isReferencedBy"); + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsRelatedTo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsRelatedTo.java index 361ea3b34..a8aa550d4 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsRelatedTo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsRelatedTo.java @@ -9,4 +9,9 @@ public class EnrichMissingPublicationIsRelatedTo extends AbstractEnrichMissingPu super(Topic.ENRICH_MISSING_PUBLICATION_IS_RELATED_TO); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isRelatedTo"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedBy.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedBy.java index 7e8863b1e..762ac942e 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedBy.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedBy.java @@ -9,4 +9,8 @@ public class EnrichMissingPublicationIsSupplementedBy extends AbstractEnrichMiss super(Topic.ENRICH_MISSING_PUBLICATION_IS_SUPPLEMENTED_BY); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isSupplementedBy"); + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedTo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedTo.java index dc4e51377..fc7196a01 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedTo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationIsSupplementedTo.java @@ -9,4 +9,9 @@ public class EnrichMissingPublicationIsSupplementedTo extends AbstractEnrichMiss super(Topic.ENRICH_MISSING_PUBLICATION_IS_SUPPLEMENTED_TO); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("isSupplementedTo"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationReferences.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationReferences.java index 5198098bc..da1994454 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationReferences.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/EnrichMissingPublicationReferences.java @@ -9,4 +9,9 @@ public class EnrichMissingPublicationReferences extends AbstractEnrichMissingPub super(Topic.ENRICH_MISSING_PUBLICATION_REFERENCES); } + @Override + protected boolean filterByType(final String relType) { + return relType.equals("references"); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java index 699d546ec..1ce5415d5 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java @@ -5,18 +5,16 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.Software; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftware; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingSoftware - extends UpdateMatcher>, eu.dnetlib.broker.objects.Software> { + extends UpdateMatcher { public EnrichMissingSoftware() { super(true); @@ -24,16 +22,17 @@ public class EnrichMissingSoftware @Override protected List> findUpdates( - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { - if (source.getRight().isEmpty()) { + if (source.getSoftwares().isEmpty()) { return Arrays.asList(); } else { return target - .getRight() + .getSoftwares() .stream() + .map(RelatedSoftware::getRelSoftware) .map(ConversionUtils::oafSoftwareToBrokerSoftware) .map(p -> generateUpdateInfo(p, source, target, dedupConfig)) .collect(Collectors.toList()); @@ -42,12 +41,12 @@ public class EnrichMissingSoftware public UpdateInfo generateUpdateInfo( final eu.dnetlib.broker.objects.Software highlightValue, - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_SOFTWARE, - highlightValue, source.getLeft(), target.getLeft(), + highlightValue, source, target, (p, s) -> p.getSoftwares().add(s), s -> s.getName(), dedupConfig); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java index 45631df20..4d1f4f23f 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java @@ -5,18 +5,17 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftware; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Software; import eu.dnetlib.pace.config.DedupConfig; public class EnrichMoreSoftware - extends UpdateMatcher>, eu.dnetlib.broker.objects.Software> { + extends UpdateMatcher { public EnrichMoreSoftware() { super(true); @@ -24,19 +23,21 @@ public class EnrichMoreSoftware @Override protected List> findUpdates( - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final Set existingSoftwares = source - .getRight() + .getSoftwares() .stream() + .map(RelatedSoftware::getRelSoftware) .map(Software::getId) .collect(Collectors.toSet()); return target - .getRight() + .getSoftwares() .stream() + .map(RelatedSoftware::getRelSoftware) .filter(p -> !existingSoftwares.contains(p.getId())) .map(ConversionUtils::oafSoftwareToBrokerSoftware) .map(p -> generateUpdateInfo(p, source, target, dedupConfig)) @@ -45,12 +46,12 @@ public class EnrichMoreSoftware public UpdateInfo generateUpdateInfo( final eu.dnetlib.broker.objects.Software highlightValue, - final Pair> source, - final Pair> target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MORE_SOFTWARE, - highlightValue, source.getLeft(), target.getLeft(), + highlightValue, source, target, (p, s) -> p.getSoftwares().add(s), s -> s.getName(), dedupConfig); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java index 7dc340b3c..db9972479 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java @@ -8,28 +8,31 @@ import java.util.List; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingAbstract extends UpdateMatcher { +public class EnrichMissingAbstract extends UpdateMatcher { public EnrichMissingAbstract() { super(false); } @Override - protected List> findUpdates(final Result source, final Result target, + protected List> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { - if (isMissing(target.getDescription()) && !isMissing(source.getDescription())) { + if (isMissing(target.getResult().getDescription()) && !isMissing(source.getResult().getDescription())) { return Arrays - .asList(generateUpdateInfo(source.getDescription().get(0).getValue(), source, target, dedupConfig)); + .asList( + generateUpdateInfo( + source.getResult().getDescription().get(0).getValue(), source, target, dedupConfig)); } return new ArrayList<>(); } public UpdateInfo generateUpdateInfo(final String highlightValue, - final Result source, - final Result target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_ABSTRACT, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java index 7a1677ae2..1226aaf45 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java @@ -9,17 +9,18 @@ import org.apache.commons.lang3.tuple.Pair; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingAuthorOrcid extends UpdateMatcher> { +public class EnrichMissingAuthorOrcid extends UpdateMatcher> { public EnrichMissingAuthorOrcid() { super(true); } @Override - protected List>> findUpdates(final Result source, final Result target, + protected List>> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { // TODO // return Arrays.asList(new EnrichMissingAbstract("xxxxxxx", 0.9f)); @@ -27,8 +28,8 @@ public class EnrichMissingAuthorOrcid extends UpdateMatcher> generateUpdateInfo(final Pair highlightValue, - final Result source, - final Result target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_AUTHOR_ORCID, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java index d14490ba8..69bd3630a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java @@ -11,19 +11,21 @@ import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingOpenAccess extends UpdateMatcher { +public class EnrichMissingOpenAccess extends UpdateMatcher { public EnrichMissingOpenAccess() { super(true); } @Override - protected List> findUpdates(final Result source, final Result target, + protected List> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final long count = target + .getResult() .getInstance() .stream() .map(i -> i.getAccessright().getClassid()) @@ -35,6 +37,7 @@ public class EnrichMissingOpenAccess extends UpdateMatcher { } return source + .getResult() .getInstance() .stream() .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) @@ -45,8 +48,8 @@ public class EnrichMissingOpenAccess extends UpdateMatcher { } public UpdateInfo generateUpdateInfo(final Instance highlightValue, - final Result source, - final Result target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_OA_VERSION, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java index 20303ec1b..4b7b1735b 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java @@ -10,25 +10,27 @@ import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingPid extends UpdateMatcher { +public class EnrichMissingPid extends UpdateMatcher { public EnrichMissingPid() { super(true); } @Override - protected List> findUpdates(final Result source, final Result target, + protected List> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { - final long count = target.getPid().size(); + final long count = target.getResult().getPid().size(); if (count > 0) { return Arrays.asList(); } return source + .getResult() .getPid() .stream() .map(ConversionUtils::oafPidToBrokerPid) @@ -36,7 +38,9 @@ public class EnrichMissingPid extends UpdateMatcher { .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo(final Pid highlightValue, final Result source, final Result target, + public UpdateInfo generateUpdateInfo(final Pid highlightValue, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_PID, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java index e1de8ce4d..ecf8da157 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java @@ -8,28 +8,32 @@ import java.util.List; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingPublicationDate extends UpdateMatcher { +public class EnrichMissingPublicationDate extends UpdateMatcher { public EnrichMissingPublicationDate() { super(false); } @Override - protected List> findUpdates(final Result source, final Result target, + protected List> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { - if (isMissing(target.getDateofacceptance()) && !isMissing(source.getDateofacceptance())) { + if (isMissing(target.getResult().getDateofacceptance()) + && !isMissing(source.getResult().getDateofacceptance())) { return Arrays - .asList(generateUpdateInfo(source.getDateofacceptance().getValue(), source, target, dedupConfig)); + .asList( + generateUpdateInfo( + source.getResult().getDateofacceptance().getValue(), source, target, dedupConfig)); } return new ArrayList<>(); } public UpdateInfo generateUpdateInfo(final String highlightValue, - final Result source, - final Result target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_PUBLICATION_DATE, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java index c51f8991c..9d3a3aa44 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java @@ -11,21 +11,23 @@ import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Qualifier; -import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingSubject extends UpdateMatcher> { +public class EnrichMissingSubject extends UpdateMatcher> { public EnrichMissingSubject() { super(true); } @Override - protected List>> findUpdates(final Result source, final Result target, + protected List>> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final Set existingTypes = target + .getResult() .getSubject() .stream() .map(StructuredProperty::getQualifier) @@ -33,6 +35,7 @@ public class EnrichMissingSubject extends UpdateMatcher !existingTypes.contains(pid.getQualifier().getClassid())) @@ -42,8 +45,8 @@ public class EnrichMissingSubject extends UpdateMatcher> generateUpdateInfo(final Pair highlightValue, - final Result source, - final Result target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java index 2ac04fd12..fc1112d73 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java @@ -11,19 +11,21 @@ import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMoreOpenAccess extends UpdateMatcher { +public class EnrichMoreOpenAccess extends UpdateMatcher { public EnrichMoreOpenAccess() { super(true); } @Override - protected List> findUpdates(final Result source, final Result target, + protected List> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final Set urls = target + .getResult() .getInstance() .stream() .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) @@ -32,6 +34,7 @@ public class EnrichMoreOpenAccess extends UpdateMatcher { .collect(Collectors.toSet()); return source + .getResult() .getInstance() .stream() .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) @@ -43,8 +46,8 @@ public class EnrichMoreOpenAccess extends UpdateMatcher { } public UpdateInfo generateUpdateInfo(final Instance highlightValue, - final Result source, - final Result target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MORE_OA_VERSION, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java index e4bf5d2c2..7984cc521 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java @@ -10,25 +10,28 @@ import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMorePid extends UpdateMatcher { +public class EnrichMorePid extends UpdateMatcher { public EnrichMorePid() { super(true); } @Override - protected List> findUpdates(final Result source, final Result target, + protected List> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final Set existingPids = target + .getResult() .getPid() .stream() .map(pid -> pid.getQualifier().getClassid() + "::" + pid.getValue()) .collect(Collectors.toSet()); return source + .getResult() .getPid() .stream() .filter(pid -> !existingPids.contains(pid.getQualifier().getClassid() + "::" + pid.getValue())) @@ -37,7 +40,9 @@ public class EnrichMorePid extends UpdateMatcher { .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo(final Pid highlightValue, final Result source, final Result target, + public UpdateInfo generateUpdateInfo(final Pid highlightValue, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MORE_PID, diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java index d6e607c31..1a522c745 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java @@ -11,25 +11,28 @@ import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMoreSubject extends UpdateMatcher> { +public class EnrichMoreSubject extends UpdateMatcher> { public EnrichMoreSubject() { super(true); } @Override - protected List>> findUpdates(final Result source, final Result target, + protected List>> findUpdates(final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { final Set existingSubjects = target + .getResult() .getSubject() .stream() .map(pid -> pid.getQualifier().getClassid() + "::" + pid.getValue()) .collect(Collectors.toSet()); return source + .getResult() .getPid() .stream() .filter(pid -> !existingSubjects.contains(pid.getQualifier().getClassid() + "::" + pid.getValue())) @@ -39,8 +42,8 @@ public class EnrichMoreSubject extends UpdateMatcher> generateUpdateInfo(final Pair highlightValue, - final Result source, - final Result target, + final ResultWithRelations source, + final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java new file mode 100644 index 000000000..b4de08db7 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java @@ -0,0 +1,86 @@ + +package eu.dnetlib.dhp.broker.oa.util; + +import java.util.ArrayList; +import java.util.List; + +import eu.dnetlib.dhp.broker.model.EventFactory; +import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsReferencedBy; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsRelatedTo; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsSupplementedBy; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsSupplementedTo; +import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetReferences; +import eu.dnetlib.dhp.broker.oa.matchers.relatedProjects.EnrichMissingProject; +import eu.dnetlib.dhp.broker.oa.matchers.relatedProjects.EnrichMoreProject; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsReferencedBy; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsRelatedTo; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsSupplementedBy; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationIsSupplementedTo; +import eu.dnetlib.dhp.broker.oa.matchers.relatedPublications.EnrichMissingPublicationReferences; +import eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware.EnrichMissingSoftware; +import eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware.EnrichMoreSoftware; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingAbstract; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingAuthorOrcid; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingOpenAccess; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPid; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingPublicationDate; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMissingSubject; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreOpenAccess; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMorePid; +import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreSubject; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; +import eu.dnetlib.pace.config.DedupConfig; + +public class EventFinder { + + private static List> matchers = new ArrayList<>(); + static { + matchers.add(new EnrichMissingAbstract()); + matchers.add(new EnrichMissingAuthorOrcid()); + matchers.add(new EnrichMissingOpenAccess()); + matchers.add(new EnrichMissingPid()); + matchers.add(new EnrichMissingPublicationDate()); + matchers.add(new EnrichMissingSubject()); + matchers.add(new EnrichMoreOpenAccess()); + matchers.add(new EnrichMorePid()); + matchers.add(new EnrichMoreSubject()); + + // Advanced matchers + matchers.add(new EnrichMissingProject()); + matchers.add(new EnrichMoreProject()); + matchers.add(new EnrichMissingSoftware()); + matchers.add(new EnrichMoreSoftware()); + matchers.add(new EnrichMissingPublicationIsRelatedTo()); + matchers.add(new EnrichMissingPublicationIsReferencedBy()); + matchers.add(new EnrichMissingPublicationReferences()); + matchers.add(new EnrichMissingPublicationIsSupplementedTo()); + matchers.add(new EnrichMissingPublicationIsSupplementedBy()); + matchers.add(new EnrichMissingDatasetIsRelatedTo()); + matchers.add(new EnrichMissingDatasetIsReferencedBy()); + matchers.add(new EnrichMissingDatasetReferences()); + matchers.add(new EnrichMissingDatasetIsSupplementedTo()); + matchers.add(new EnrichMissingDatasetIsSupplementedBy()); + matchers.add(new EnrichMissingAbstract()); + } + + public static EventGroup generateEvents(final ResultGroup results, final DedupConfig dedupConfig) { + final List> list = new ArrayList<>(); + + for (final ResultWithRelations target : results.getData()) { + for (final UpdateMatcher matcher : matchers) { + list.addAll(matcher.searchUpdatesForRecord(target, results.getData(), dedupConfig)); + } + } + + return asEventGroup(list); + } + + private static EventGroup asEventGroup(final List> list) { + final EventGroup events = new EventGroup(); + list.stream().map(EventFactory::newBrokerEvent).forEach(events::addElement); + return events; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java index 893aa2827..9da636413 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java @@ -14,6 +14,7 @@ import eu.dnetlib.broker.objects.OpenAireEventPayload; import eu.dnetlib.broker.objects.Provenance; import eu.dnetlib.broker.objects.Publication; import eu.dnetlib.dhp.broker.model.Topic; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Instance; import eu.dnetlib.dhp.schema.oaf.KeyValue; import eu.dnetlib.dhp.schema.oaf.Result; @@ -28,9 +29,9 @@ public final class UpdateInfo { private final T highlightValue; - private final Result source; + private final ResultWithRelations source; - private final Result target; + private final ResultWithRelations target; private final BiConsumer compileHighlight; @@ -40,7 +41,8 @@ public final class UpdateInfo { private static final Logger log = LoggerFactory.getLogger(UpdateInfo.class); - public UpdateInfo(final Topic topic, final T highlightValue, final Result source, final Result target, + public UpdateInfo(final Topic topic, final T highlightValue, final ResultWithRelations source, + final ResultWithRelations target, final BiConsumer compileHighlight, final Function highlightToString, final DedupConfig dedupConfig) { @@ -50,18 +52,18 @@ public final class UpdateInfo { this.target = target; this.compileHighlight = compileHighlight; this.highlightToString = highlightToString; - this.trust = calculateTrust(dedupConfig, source, target); + this.trust = calculateTrust(dedupConfig, source.getResult(), target.getResult()); } public T getHighlightValue() { return highlightValue; } - public Result getSource() { + public ResultWithRelations getSource() { return source; } - public Result getTarget() { + public ResultWithRelations getTarget() { return target; } @@ -101,20 +103,22 @@ public final class UpdateInfo { public OpenAireEventPayload asBrokerPayload() { - final Publication p = ConversionUtils.oafResultToBrokerPublication(getSource()); + final Publication p = ConversionUtils.oafResultToBrokerPublication(getSource().getResult()); compileHighlight.accept(p, getHighlightValue()); final Publication hl = new Publication(); compileHighlight.accept(hl, getHighlightValue()); - final String provId = getSource().getOriginalId().stream().findFirst().orElse(null); + final String provId = getSource().getResult().getOriginalId().stream().findFirst().orElse(null); final String provRepo = getSource() + .getResult() .getCollectedfrom() .stream() .map(KeyValue::getValue) .findFirst() .orElse(null); final String provUrl = getSource() + .getResult() .getInstance() .stream() .map(Instance::getUrl) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java similarity index 75% rename from dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultAggregator.java rename to dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java index 475c76814..397f30660 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultAggregator.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java @@ -1,15 +1,15 @@ -package eu.dnetlib.dhp.broker.oa.util; +package eu.dnetlib.dhp.broker.oa.util.aggregators.simple; import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.expressions.Aggregator; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Relation; -import eu.dnetlib.dhp.schema.oaf.Result; import scala.Tuple2; -public class ResultAggregator extends Aggregator, ResultGroup, ResultGroup> { +public class ResultAggregator extends Aggregator, ResultGroup, ResultGroup> { /** * @@ -22,7 +22,7 @@ public class ResultAggregator extends Aggregator, Resul } @Override - public ResultGroup reduce(final ResultGroup group, final Tuple2 t) { + public ResultGroup reduce(final ResultGroup group, final Tuple2 t) { return group.addElement(t._1); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultGroup.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java similarity index 58% rename from dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultGroup.java rename to dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java index 2be673db0..81f189d62 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ResultGroup.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java @@ -1,11 +1,11 @@ -package eu.dnetlib.dhp.broker.oa.util; +package eu.dnetlib.dhp.broker.oa.util.aggregators.simple; import java.io.Serializable; import java.util.ArrayList; import java.util.List; -import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; public class ResultGroup implements Serializable { @@ -14,13 +14,13 @@ public class ResultGroup implements Serializable { */ private static final long serialVersionUID = -3360828477088669296L; - private final List data = new ArrayList<>(); + private final List data = new ArrayList<>(); - public List getData() { + public List getData() { return data; } - public ResultGroup addElement(final Result elem) { + public ResultGroup addElement(final ResultWithRelations elem) { data.add(elem); return this; } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java new file mode 100644 index 000000000..84cf693ad --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java @@ -0,0 +1,30 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import eu.dnetlib.dhp.schema.oaf.Dataset; + +public class RelatedDataset { + + private final String source; + private final String relType; + private final Dataset relDataset; + + public RelatedDataset(final String source, final String relType, final Dataset relDataset) { + this.source = source; + this.relType = relType; + this.relDataset = relDataset; + } + + public String getSource() { + return source; + } + + public String getRelType() { + return relType; + } + + public Dataset getRelDataset() { + return relDataset; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java new file mode 100644 index 000000000..490724a44 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java @@ -0,0 +1,28 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import eu.dnetlib.dhp.schema.oaf.Dataset; +import eu.dnetlib.dhp.schema.oaf.Project; +import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Software; + +public class RelatedEntityFactory { + + @SuppressWarnings("unchecked") + public static RT newRelatedEntity(final String sourceId, final String relType, final T target, + final Class clazz) { + if (clazz == RelatedProject.class) { + return (RT) new RelatedProject(sourceId, relType, (Project) target); + } + if (clazz == RelatedSoftware.class) { + return (RT) new RelatedSoftware(sourceId, relType, (Software) target); + } + if (clazz == RelatedDataset.class) { + return (RT) new RelatedDataset(sourceId, relType, (Dataset) target); + } + if (clazz == RelatedPublication.class) { + return (RT) new RelatedPublication(sourceId, relType, (Publication) target); + } + return null; + } +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java new file mode 100644 index 000000000..2d16f5409 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java @@ -0,0 +1,30 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import eu.dnetlib.dhp.schema.oaf.Project; + +public class RelatedProject { + + private final String source; + private final String relType; + private final Project relProject; + + public RelatedProject(final String source, final String relType, final Project relProject) { + this.source = source; + this.relType = relType; + this.relProject = relProject; + } + + public String getSource() { + return source; + } + + public String getRelType() { + return relType; + } + + public Project getRelProject() { + return relProject; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java new file mode 100644 index 000000000..f1545c004 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java @@ -0,0 +1,30 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import eu.dnetlib.dhp.schema.oaf.Publication; + +public class RelatedPublication { + + private final String source; + private final String relType; + private final Publication relPublication; + + public RelatedPublication(final String source, final String relType, final Publication relPublication) { + this.source = source; + this.relType = relType; + this.relPublication = relPublication; + } + + public String getSource() { + return source; + } + + public String getRelType() { + return relType; + } + + public Publication getRelPublication() { + return relPublication; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java new file mode 100644 index 000000000..e5873d263 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java @@ -0,0 +1,30 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import eu.dnetlib.dhp.schema.oaf.Software; + +public class RelatedSoftware { + + private final String source; + private final String relType; + private final Software relSoftware; + + public RelatedSoftware(final String source, final String relType, final Software relSoftware) { + this.source = source; + this.relType = relType; + this.relSoftware = relSoftware; + } + + public String getSource() { + return source; + } + + public String getRelType() { + return relType; + } + + public Software getRelSoftware() { + return relSoftware; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelations.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelations.java new file mode 100644 index 000000000..2d762aded --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelations.java @@ -0,0 +1,55 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import eu.dnetlib.dhp.schema.oaf.Result; + +public class ResultWithRelations implements Serializable { + + /** + * + */ + private static final long serialVersionUID = -1368401915974311571L; + + private Result result; + + private final List datasets = new ArrayList<>(); + private final List publications = new ArrayList<>(); + private final List softwares = new ArrayList<>(); + private final List projects = new ArrayList<>(); + + public ResultWithRelations() { + } + + public ResultWithRelations(final Result result) { + this.result = result; + } + + public Result getResult() { + return result; + } + + public List getDatasets() { + return datasets; + } + + public List getPublications() { + return publications; + } + + public List getSoftwares() { + return softwares; + } + + public List getProjects() { + return projects; + } + + public void setResult(final Result result) { + this.result = result; + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelationsAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelationsAggregator.java new file mode 100644 index 000000000..b4922a64f --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelationsAggregator.java @@ -0,0 +1,68 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.expressions.Aggregator; + +import scala.Tuple2; + +public class ResultWithRelationsAggregator + extends Aggregator, ResultWithRelations, ResultWithRelations> { + + /** + * + */ + private static final long serialVersionUID = -3687878788861013488L; + + @Override + public ResultWithRelations zero() { + return new ResultWithRelations(); + } + + @Override + public ResultWithRelations finish(final ResultWithRelations g) { + return g; + } + + @Override + public ResultWithRelations reduce(final ResultWithRelations g, final Tuple2 t) { + if (g.getResult() == null) { + return t._1; + } else if (t._2 instanceof RelatedSoftware) { + g.getSoftwares().add((RelatedSoftware) t._2); + } else if (t._2 instanceof RelatedDataset) { + g.getDatasets().add((RelatedDataset) t._2); + } else if (t._2 instanceof RelatedPublication) { + g.getPublications().add((RelatedPublication) t._2); + } else if (t._2 instanceof RelatedProject) { + g.getProjects().add((RelatedProject) t._2); + } + return g; + + } + + @Override + public ResultWithRelations merge(final ResultWithRelations g1, final ResultWithRelations g2) { + if (g1.getResult() != null) { + g1.getSoftwares().addAll(g2.getSoftwares()); + g1.getDatasets().addAll(g2.getDatasets()); + g1.getPublications().addAll(g2.getPublications()); + g1.getProjects().addAll(g2.getProjects()); + return g1; + } else { + return g2; + } + } + + @Override + public Encoder bufferEncoder() { + return Encoders.kryo(ResultWithRelations.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.kryo(ResultWithRelations.class); + } + +} From fa8c5bcd390d7283f10b03e36fc4e67d17592839 Mon Sep 17 00:00:00 2001 From: miconis Date: Thu, 11 Jun 2020 12:19:32 +0200 Subject: [PATCH 60/88] javadoc for the PacePerson class and implementation of a unit test --- .../eu/dnetlib/dhp/common/PacePerson.java | 31 +++++++++++++++++-- .../eu/dnetlib/dhp/common/PacePersonTest.java | 26 ++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java index 1909ddcca..ccd42225a 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java @@ -16,6 +16,13 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.hash.Hashing; +/** +* PacePerson tries to derive information from the fullname string of an author. +* Such informations are Names, Surnames an Fullname split into terms. It provides also an additional field for +* the original data. +* The calculation of the names and the surnames is not always possible. When it is impossible to assert which are the +* names and the surnames, the lists are empty. +* */ public class PacePerson { private static final String UTF8 = "UTF-8"; @@ -26,10 +33,19 @@ public class PacePerson { private static Set particles = null; + /** + * Capitalizes a string + * + * @param s the string to capitalize + * @return the input string with capital letter + * */ public static final String capitalize(final String s) { return WordUtils.capitalize(s.toLowerCase(), ' ', '-'); } + /** + * Adds a dot to a string with length equals to 1 + * */ public static final String dotAbbreviations(final String s) { return s.length() == 1 ? s + "." : s; } @@ -46,6 +62,12 @@ public class PacePerson { return h; } + /** + * The constructor of the class. It fills the fields of the class basing on the input fullname. + * + * @param s the input string (fullname of the author) + * @param aggressive set the string normalization type + * */ public PacePerson(String s, final boolean aggressive) { original = s; s = Normalizer.normalize(s, Normalizer.Form.NFD); @@ -64,6 +86,7 @@ public class PacePerson { // s = s.replaceAll("[\\W&&[^,-]]", ""); } + //if the string contains a comma, it can derive surname and name by splitting on it if (s.contains(",")) { final String[] arr = s.split(","); if (arr.length == 1) { @@ -74,21 +97,23 @@ public class PacePerson { fullname.addAll(surname); fullname.addAll(name); } - } else { + } else { //otherwise, it should rely on CAPS terms and short terms fullname = splitTerms(s); int lastInitialPosition = fullname.size(); boolean hasSurnameInUpperCase = false; + //computes lastInitialPosition and hasSurnameInUpperCase for (int i = 0; i < fullname.size(); i++) { final String term = fullname.get(i); if (term.length() == 1) { - lastInitialPosition = i; + lastInitialPosition = i; //first word in the name longer than 1 (to avoid name with dots) } else if (term.equals(term.toUpperCase())) { - hasSurnameInUpperCase = true; + hasSurnameInUpperCase = true; //if one of the words is CAPS } } + //manages particular cases of fullnames if (lastInitialPosition < fullname.size() - 1) { // Case: Michele G. Artini name = fullname.subList(0, lastInitialPosition + 1); surname = fullname.subList(lastInitialPosition + 1, fullname.size()); diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java new file mode 100644 index 000000000..7ee60a0aa --- /dev/null +++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java @@ -0,0 +1,26 @@ +package eu.dnetlib.dhp.common; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class PacePersonTest { + + @Test + public void pacePersonTest1(){ + + PacePerson p = new PacePerson("Artini, Michele", false); + assertEquals("Artini",p.getSurnameString()); + assertEquals("Michele", p.getNameString()); + assertEquals("Artini, Michele", p.getNormalisedFullname()); + } + + @Test + public void pacePersonTest2(){ + PacePerson p = new PacePerson("Michele G. Artini", false); + assertEquals("Artini, Michele G.", p.getNormalisedFullname()); + assertEquals("Michele G", p.getNameString()); + assertEquals("Artini", p.getSurnameString()); + } + +} From a41e0cb64899dc3673dd119e93a2550b4b8c70ec Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 11 Jun 2020 12:28:34 +0200 Subject: [PATCH 61/88] missing landingPage urls in instances --- .../eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index e6a744fc0..7ff483aff 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -138,9 +138,16 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { for (final Object o : doc.selectNodes("//datacite:alternateIdentifier[@alternateIdentifierType='URL']")) { url.add(((Node) o).getText().trim()); } + for (final Object o : doc + .selectNodes("//datacite:alternateIdentifier[@alternateIdentifierType='landingPage']")) { + url.add(((Node) o).getText().trim()); + } for (final Object o : doc.selectNodes("//datacite:identifier[@identifierType='URL']")) { url.add(((Node) o).getText().trim()); } + for (final Object o : doc.selectNodes("//datacite:identifier[@identifierType='landingPage']")) { + url.add(((Node) o).getText().trim()); + } for (final Object o : doc.selectNodes("//datacite:alternateIdentifier[@alternateIdentifierType='DOI']")) { url.add(HTTP_DX_DOI_PREIFX + ((Node) o).getText().trim()); } @@ -379,11 +386,13 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { res .addAll( prepareListStructPropsWithValidQualifier( - doc, "//datacite:identifier[@identifierType != 'URL']", "@identifierType", DNET_PID_TYPES, info)); + doc, "//datacite:identifier[@identifierType != 'URL' and @identifierType != 'landingPage']", + "@identifierType", DNET_PID_TYPES, info)); res .addAll( prepareListStructPropsWithValidQualifier( - doc, "//datacite:alternateIdentifier[@alternateIdentifierType != 'URL']", + doc, + "//datacite:alternateIdentifier[@alternateIdentifierType != 'URL' and @alternateIdentifierType != 'landingPage']", "@alternateIdentifierType", DNET_PID_TYPES, info)); return res; } From e79943965b23e36936889613dc2ba8b7691f1740 Mon Sep 17 00:00:00 2001 From: Alessia Bardi Date: Thu, 11 Jun 2020 12:49:31 +0200 Subject: [PATCH 62/88] Fixes #5604: field oamandatepublications in XML --- .../eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java index d950a816d..21ffd6992 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java @@ -896,6 +896,9 @@ public class XmlRecordFactory implements Serializable { if (p.getContracttype() != null) { metadata.add(XmlSerializationUtils.mapQualifier("contracttype", p.getContracttype())); } + if (p.getOamandatepublications() != null) { + metadata.add(XmlSerializationUtils.asXmlElement("oamandatepublications", p.getOamandatepublications().getValue())); + } if (p.getEcsc39() != null) { metadata.add(XmlSerializationUtils.asXmlElement("ecsc39", p.getEcsc39().getValue())); } From 48959e9a1710f627a902dc986e07e180193edb9e Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 11 Jun 2020 14:24:02 +0200 Subject: [PATCH 63/88] orcid events --- .../simple/EnrichMissingAuthorOrcid.java | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java index 1226aaf45..14021480d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java @@ -1,41 +1,68 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; - -import org.apache.commons.lang3.tuple.Pair; +import java.util.Set; +import java.util.stream.Collectors; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; +import eu.dnetlib.dhp.schema.oaf.Author; +import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingAuthorOrcid extends UpdateMatcher> { +public class EnrichMissingAuthorOrcid extends UpdateMatcher { public EnrichMissingAuthorOrcid() { super(true); } @Override - protected List>> findUpdates(final ResultWithRelations source, + protected List> findUpdates(final ResultWithRelations source, final ResultWithRelations target, final DedupConfig dedupConfig) { - // TODO - // return Arrays.asList(new EnrichMissingAbstract("xxxxxxx", 0.9f)); - return Arrays.asList(); + + final Set existingOrcids = target + .getResult() + .getAuthor() + .stream() + .map(Author::getPid) + .flatMap(List::stream) + .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) + .map(pid -> pid.getValue()) + .collect(Collectors.toSet()); + + final List> list = new ArrayList<>(); + + for (final Author author : source.getResult().getAuthor()) { + final String name = author.getFullname(); + + for (final StructuredProperty pid : author.getPid()) { + if (pid.getQualifier().getClassid().equalsIgnoreCase("orcid") + && !existingOrcids.contains(pid.getValue())) { + list + .add( + generateUpdateInfo(name + " [ORCID: " + pid.getValue() + "]", source, target, dedupConfig)); + ; + } + } + } + + return list; } - public UpdateInfo> generateUpdateInfo(final Pair highlightValue, + public UpdateInfo generateUpdateInfo(final String highlightValue, final ResultWithRelations source, final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_AUTHOR_ORCID, highlightValue, source, target, - (p, pair) -> p.getCreators().add(pair.getLeft() + " - ORCID: " + pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight(), + (p, aut) -> p.getCreators().add(aut), + aut -> aut, dedupConfig); } } From c2e1b66e83b6bda2cdf863a1c10e6e02411baea1 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 11 Jun 2020 14:28:03 +0200 Subject: [PATCH 64/88] Revert "orcid events" This reverts commit 48959e9a1710f627a902dc986e07e180193edb9e. --- .../simple/EnrichMissingAuthorOrcid.java | 49 +++++-------------- 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java index 14021480d..1226aaf45 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java @@ -1,68 +1,41 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; -import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; + +import org.apache.commons.lang3.tuple.Pair; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Author; -import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingAuthorOrcid extends UpdateMatcher { +public class EnrichMissingAuthorOrcid extends UpdateMatcher> { public EnrichMissingAuthorOrcid() { super(true); } @Override - protected List> findUpdates(final ResultWithRelations source, + protected List>> findUpdates(final ResultWithRelations source, final ResultWithRelations target, final DedupConfig dedupConfig) { - - final Set existingOrcids = target - .getResult() - .getAuthor() - .stream() - .map(Author::getPid) - .flatMap(List::stream) - .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) - .map(pid -> pid.getValue()) - .collect(Collectors.toSet()); - - final List> list = new ArrayList<>(); - - for (final Author author : source.getResult().getAuthor()) { - final String name = author.getFullname(); - - for (final StructuredProperty pid : author.getPid()) { - if (pid.getQualifier().getClassid().equalsIgnoreCase("orcid") - && !existingOrcids.contains(pid.getValue())) { - list - .add( - generateUpdateInfo(name + " [ORCID: " + pid.getValue() + "]", source, target, dedupConfig)); - ; - } - } - } - - return list; + // TODO + // return Arrays.asList(new EnrichMissingAbstract("xxxxxxx", 0.9f)); + return Arrays.asList(); } - public UpdateInfo generateUpdateInfo(final String highlightValue, + public UpdateInfo> generateUpdateInfo(final Pair highlightValue, final ResultWithRelations source, final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_AUTHOR_ORCID, highlightValue, source, target, - (p, aut) -> p.getCreators().add(aut), - aut -> aut, + (p, pair) -> p.getCreators().add(pair.getLeft() + " - ORCID: " + pair.getRight()), + pair -> pair.getLeft() + "::" + pair.getRight(), dedupConfig); } } From c6b5bb3f173343f3348500ab6db1f0766cb2daec Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 11 Jun 2020 14:30:24 +0200 Subject: [PATCH 65/88] orcid events --- .../simple/EnrichMissingAuthorOrcid.java | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java index 1226aaf45..14021480d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java @@ -1,41 +1,68 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; - -import org.apache.commons.lang3.tuple.Pair; +import java.util.Set; +import java.util.stream.Collectors; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; +import eu.dnetlib.dhp.schema.oaf.Author; +import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.pace.config.DedupConfig; -public class EnrichMissingAuthorOrcid extends UpdateMatcher> { +public class EnrichMissingAuthorOrcid extends UpdateMatcher { public EnrichMissingAuthorOrcid() { super(true); } @Override - protected List>> findUpdates(final ResultWithRelations source, + protected List> findUpdates(final ResultWithRelations source, final ResultWithRelations target, final DedupConfig dedupConfig) { - // TODO - // return Arrays.asList(new EnrichMissingAbstract("xxxxxxx", 0.9f)); - return Arrays.asList(); + + final Set existingOrcids = target + .getResult() + .getAuthor() + .stream() + .map(Author::getPid) + .flatMap(List::stream) + .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) + .map(pid -> pid.getValue()) + .collect(Collectors.toSet()); + + final List> list = new ArrayList<>(); + + for (final Author author : source.getResult().getAuthor()) { + final String name = author.getFullname(); + + for (final StructuredProperty pid : author.getPid()) { + if (pid.getQualifier().getClassid().equalsIgnoreCase("orcid") + && !existingOrcids.contains(pid.getValue())) { + list + .add( + generateUpdateInfo(name + " [ORCID: " + pid.getValue() + "]", source, target, dedupConfig)); + ; + } + } + } + + return list; } - public UpdateInfo> generateUpdateInfo(final Pair highlightValue, + public UpdateInfo generateUpdateInfo(final String highlightValue, final ResultWithRelations source, final ResultWithRelations target, final DedupConfig dedupConfig) { return new UpdateInfo<>( Topic.ENRICH_MISSING_AUTHOR_ORCID, highlightValue, source, target, - (p, pair) -> p.getCreators().add(pair.getLeft() + " - ORCID: " + pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight(), + (p, aut) -> p.getCreators().add(aut), + aut -> aut, dedupConfig); } } From c22cb5a3c6575daa656aefa493853f2f787cd52f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 12 Jun 2020 09:47:55 +0200 Subject: [PATCH 66/88] refactoring --- .../dhp/broker/oa/matchers/UpdateMatcher.java | 39 +++++++++++++++++-- .../AbstractEnrichMissingDataset.java | 33 +++------------- .../relatedProjects/EnrichMissingProject.java | 27 +++---------- .../relatedProjects/EnrichMoreProject.java | 25 +++--------- .../AbstractEnrichMissingPublication.java | 33 ++++------------ .../EnrichMissingSoftware.java | 25 +++--------- .../relatedSoftware/EnrichMoreSoftware.java | 25 +++--------- .../simple/EnrichMissingAbstract.java | 27 ++++--------- .../simple/EnrichMissingAuthorOrcid.java | 31 ++++----------- .../simple/EnrichMissingOpenAccess.java | 24 +++--------- .../oa/matchers/simple/EnrichMissingPid.java | 24 +++--------- .../simple/EnrichMissingPublicationDate.java | 28 ++++--------- .../matchers/simple/EnrichMissingSubject.java | 25 +++--------- .../matchers/simple/EnrichMoreOpenAccess.java | 24 +++--------- .../oa/matchers/simple/EnrichMorePid.java | 24 +++--------- .../oa/matchers/simple/EnrichMoreSubject.java | 25 +++--------- 16 files changed, 130 insertions(+), 309 deletions(-) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java index fd87d81dd..0e57f32f9 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java @@ -6,10 +6,14 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; +import eu.dnetlib.broker.objects.Publication; +import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Field; @@ -18,9 +22,17 @@ import eu.dnetlib.pace.config.DedupConfig; public abstract class UpdateMatcher { private final boolean multipleUpdate; + private final Function topicFunction; + private final BiConsumer compileHighlightFunction; + private final Function highlightToStringFunction; - public UpdateMatcher(final boolean multipleUpdate) { + public UpdateMatcher(final boolean multipleUpdate, final Function topicFunction, + final BiConsumer compileHighlightFunction, + final Function highlightToStringFunction) { this.multipleUpdate = multipleUpdate; + this.topicFunction = topicFunction; + this.compileHighlightFunction = compileHighlightFunction; + this.highlightToStringFunction = highlightToStringFunction; } public Collection> searchUpdatesForRecord(final ResultWithRelations res, @@ -31,7 +43,11 @@ public abstract class UpdateMatcher { for (final ResultWithRelations source : others) { if (source != res) { - for (final UpdateInfo info : findUpdates(source, res, dedupConfig)) { + for (final T hl : findDifferences(source, res)) { + final Topic topic = getTopicFunction().apply(hl); + final UpdateInfo info = new UpdateInfo<>(topic, hl, source, res, getCompileHighlightFunction(), + getHighlightToStringFunction(), + dedupConfig); final String s = DigestUtils.md5Hex(info.getHighlightValueAsString()); if (!infoMap.containsKey(s) || infoMap.get(s).getTrust() < info.getTrust()) { } else { @@ -55,8 +71,7 @@ public abstract class UpdateMatcher { } } - protected abstract List> findUpdates(ResultWithRelations source, ResultWithRelations target, - DedupConfig dedupConfig); + protected abstract List findDifferences(ResultWithRelations source, ResultWithRelations target); protected static boolean isMissing(final List> list) { return list == null || list.isEmpty() || StringUtils.isBlank(list.get(0).getValue()); @@ -66,4 +81,20 @@ public abstract class UpdateMatcher { return field == null || StringUtils.isBlank(field.getValue()); } + public boolean isMultipleUpdate() { + return multipleUpdate; + } + + public Function getTopicFunction() { + return topicFunction; + } + + public BiConsumer getCompileHighlightFunction() { + return compileHighlightFunction; + } + + public Function getHighlightToStringFunction() { + return highlightToStringFunction; + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java index a2ce32a9d..a8e0a2c42 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java @@ -8,29 +8,26 @@ import java.util.stream.Collectors; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedDataset; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Dataset; -import eu.dnetlib.pace.config.DedupConfig; public abstract class AbstractEnrichMissingDataset extends UpdateMatcher { - private final Topic topic; - public AbstractEnrichMissingDataset(final Topic topic) { - super(true); - this.topic = topic; + super(true, + rel -> topic, + (p, rel) -> p.getDatasets().add(rel), + rel -> rel.getInstances().get(0).getUrl()); } protected abstract boolean filterByType(String relType); @Override - protected final List> findUpdates( + protected final List findDifferences( final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + final ResultWithRelations target) { final Set existingDatasets = target .getDatasets() @@ -47,26 +44,8 @@ public abstract class AbstractEnrichMissingDataset .map(RelatedDataset::getRelDataset) .filter(d -> !existingDatasets.contains(d.getId())) .map(ConversionUtils::oafDatasetToBrokerDataset) - .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) .collect(Collectors.toList()); } - protected final UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Dataset highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - getTopic(), - highlightValue, source, target, - (p, rel) -> p.getDatasets().add(rel), - rel -> rel.getInstances().get(0).getUrl(), - dedupConfig); - } - - public Topic getTopic() { - return topic; - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java index 546287795..ab4325c1e 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java @@ -5,26 +5,25 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import eu.dnetlib.broker.objects.Project; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingProject extends UpdateMatcher { public EnrichMissingProject() { - super(true); + super(true, + prj -> Topic.ENRICH_MISSING_PROJECT, + (p, prj) -> p.getProjects().add(prj), + prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode()); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - + protected List findDifferences(final ResultWithRelations source, final ResultWithRelations target) { if (source.getProjects().isEmpty()) { return Arrays.asList(); } else { @@ -33,21 +32,7 @@ public class EnrichMissingProject .stream() .map(RelatedProject::getRelProject) .map(ConversionUtils::oafProjectToBrokerProject) - .map(p -> generateUpdateInfo(p, source, target, dedupConfig)) .collect(Collectors.toList()); } } - - public UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Project highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_PROJECT, - highlightValue, source, target, - (p, prj) -> p.getProjects().add(prj), - prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode(), dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java index 54ebe7b71..3bf23a36b 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java @@ -8,22 +8,22 @@ import java.util.stream.Collectors; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Project; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMoreProject extends UpdateMatcher { public EnrichMoreProject() { - super(true); + super(true, + prj -> Topic.ENRICH_MORE_PROJECT, + (p, prj) -> p.getProjects().add(prj), + prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode()); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { final Set existingProjects = source .getProjects() @@ -38,20 +38,7 @@ public class EnrichMoreProject extends UpdateMatcher !existingProjects.contains(p.getId())) .map(ConversionUtils::oafProjectToBrokerProject) - .map(p -> generateUpdateInfo(p, source, target, dedupConfig)) .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Project highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MORE_PROJECT, - highlightValue, source, target, - (p, prj) -> p.getProjects().add(prj), - prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode(), dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java index 8793d38dc..bba3e9648 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java @@ -8,29 +8,27 @@ import java.util.stream.Collectors; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedPublication; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.pace.config.DedupConfig; public abstract class AbstractEnrichMissingPublication extends UpdateMatcher { - private final Topic topic; - public AbstractEnrichMissingPublication(final Topic topic) { - super(true); - this.topic = topic; + super(true, + rel -> topic, + (p, rel) -> p.getPublications().add(rel), + rel -> rel.getInstances().get(0).getUrl()); + } protected abstract boolean filterByType(String relType); @Override - protected final List> findUpdates( + protected final List findDifferences( final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + final ResultWithRelations target) { final Set existingPublications = target .getPublications() @@ -47,24 +45,7 @@ public abstract class AbstractEnrichMissingPublication .map(RelatedPublication::getRelPublication) .filter(d -> !existingPublications.contains(d.getId())) .map(ConversionUtils::oafResultToBrokerPublication) - .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) .collect(Collectors.toList()); - } - protected final UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Publication highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - getTopic(), - highlightValue, source, target, - (p, rel) -> p.getPublications().add(rel), - rel -> rel.getInstances().get(0).getUrl(), dedupConfig); - } - - public Topic getTopic() { - return topic; - } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java index 1ce5415d5..6090939dc 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java @@ -8,23 +8,23 @@ import java.util.stream.Collectors; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftware; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingSoftware extends UpdateMatcher { public EnrichMissingSoftware() { - super(true); + super(true, + s -> Topic.ENRICH_MISSING_SOFTWARE, + (p, s) -> p.getSoftwares().add(s), + s -> s.getName()); } @Override - protected List> findUpdates( + protected List findDifferences( final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + final ResultWithRelations target) { if (source.getSoftwares().isEmpty()) { return Arrays.asList(); @@ -34,21 +34,8 @@ public class EnrichMissingSoftware .stream() .map(RelatedSoftware::getRelSoftware) .map(ConversionUtils::oafSoftwareToBrokerSoftware) - .map(p -> generateUpdateInfo(p, source, target, dedupConfig)) .collect(Collectors.toList()); } } - public UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Software highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_SOFTWARE, - highlightValue, source, target, - (p, s) -> p.getSoftwares().add(s), - s -> s.getName(), dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java index 4d1f4f23f..ba422f436 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java @@ -8,24 +8,24 @@ import java.util.stream.Collectors; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftware; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Software; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMoreSoftware extends UpdateMatcher { public EnrichMoreSoftware() { - super(true); + super(true, + s -> Topic.ENRICH_MORE_SOFTWARE, + (p, s) -> p.getSoftwares().add(s), + s -> s.getName()); } @Override - protected List> findUpdates( + protected List findDifferences( final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + final ResultWithRelations target) { final Set existingSoftwares = source .getSoftwares() @@ -40,20 +40,7 @@ public class EnrichMoreSoftware .map(RelatedSoftware::getRelSoftware) .filter(p -> !existingSoftwares.contains(p.getId())) .map(ConversionUtils::oafSoftwareToBrokerSoftware) - .map(p -> generateUpdateInfo(p, source, target, dedupConfig)) .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo( - final eu.dnetlib.broker.objects.Software highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MORE_SOFTWARE, - highlightValue, source, target, - (p, s) -> p.getSoftwares().add(s), - s -> s.getName(), dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java index db9972479..25d5f9d8a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java @@ -7,38 +7,25 @@ import java.util.List; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingAbstract extends UpdateMatcher { public EnrichMissingAbstract() { - super(false); + super(false, + s -> Topic.ENRICH_MISSING_ABSTRACT, + (p, s) -> p.getAbstracts().add(s), + s -> s); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { if (isMissing(target.getResult().getDescription()) && !isMissing(source.getResult().getDescription())) { return Arrays - .asList( - generateUpdateInfo( - source.getResult().getDescription().get(0).getValue(), source, target, dedupConfig)); + .asList(source.getResult().getDescription().get(0).getValue()); } return new ArrayList<>(); } - public UpdateInfo generateUpdateInfo(final String highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_ABSTRACT, - highlightValue, source, target, - (p, s) -> p.getAbstracts().add(s), - s -> s, dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java index 14021480d..eed2d0be8 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java @@ -8,22 +8,22 @@ import java.util.stream.Collectors; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingAuthorOrcid extends UpdateMatcher { public EnrichMissingAuthorOrcid() { - super(true); + super(true, + aut -> Topic.ENRICH_MISSING_AUTHOR_ORCID, + (p, aut) -> p.getCreators().add(aut), + aut -> aut); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { final Set existingOrcids = target .getResult() @@ -35,7 +35,7 @@ public class EnrichMissingAuthorOrcid extends UpdateMatcher { .map(pid -> pid.getValue()) .collect(Collectors.toSet()); - final List> list = new ArrayList<>(); + final List list = new ArrayList<>(); for (final Author author : source.getResult().getAuthor()) { final String name = author.getFullname(); @@ -43,26 +43,11 @@ public class EnrichMissingAuthorOrcid extends UpdateMatcher { for (final StructuredProperty pid : author.getPid()) { if (pid.getQualifier().getClassid().equalsIgnoreCase("orcid") && !existingOrcids.contains(pid.getValue())) { - list - .add( - generateUpdateInfo(name + " [ORCID: " + pid.getValue() + "]", source, target, dedupConfig)); - ; + list.add(name + " [ORCID: " + pid.getValue() + "]"); } } } return list; } - - public UpdateInfo generateUpdateInfo(final String highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_AUTHOR_ORCID, - highlightValue, source, target, - (p, aut) -> p.getCreators().add(aut), - aut -> aut, - dedupConfig); - } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java index 69bd3630a..e8ca18ae9 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java @@ -10,20 +10,20 @@ import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingOpenAccess extends UpdateMatcher { public EnrichMissingOpenAccess() { - super(true); + super(true, + i -> Topic.ENRICH_MISSING_OA_VERSION, + (p, i) -> p.getInstances().add(i), + Instance::getUrl); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { final long count = target .getResult() .getInstance() @@ -43,19 +43,7 @@ public class EnrichMissingOpenAccess extends UpdateMatcher { .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) .map(ConversionUtils::oafInstanceToBrokerInstances) .flatMap(List::stream) - .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo(final Instance highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_OA_VERSION, - highlightValue, source, target, - (p, i) -> p.getInstances().add(i), - Instance::getUrl, dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java index 4b7b1735b..6c061a2e8 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java @@ -9,20 +9,20 @@ import eu.dnetlib.broker.objects.Pid; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingPid extends UpdateMatcher { public EnrichMissingPid() { - super(true); + super(true, + pid -> Topic.ENRICH_MISSING_PID, + (p, pid) -> p.getPids().add(pid), + pid -> pid.getType() + "::" + pid.getValue()); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { final long count = target.getResult().getPid().size(); if (count > 0) { @@ -34,19 +34,7 @@ public class EnrichMissingPid extends UpdateMatcher { .getPid() .stream() .map(ConversionUtils::oafPidToBrokerPid) - .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo(final Pid highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_PID, - highlightValue, source, target, - (p, pid) -> p.getPids().add(pid), - pid -> pid.getType() + "::" + pid.getValue(), dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java index ecf8da157..27a71740c 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java @@ -7,39 +7,25 @@ import java.util.List; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingPublicationDate extends UpdateMatcher { public EnrichMissingPublicationDate() { - super(false); + super(false, + date -> Topic.ENRICH_MISSING_PUBLICATION_DATE, + (p, date) -> p.setPublicationdate(date), + s -> s); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { if (isMissing(target.getResult().getDateofacceptance()) && !isMissing(source.getResult().getDateofacceptance())) { - return Arrays - .asList( - generateUpdateInfo( - source.getResult().getDateofacceptance().getValue(), source, target, dedupConfig)); + return Arrays.asList(source.getResult().getDateofacceptance().getValue()); } return new ArrayList<>(); } - public UpdateInfo generateUpdateInfo(final String highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MISSING_PUBLICATION_DATE, - highlightValue, source, target, - (p, date) -> p.setPublicationdate(date), - s -> s, dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java index 9d3a3aa44..04de30089 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java @@ -10,22 +10,22 @@ import org.apache.commons.lang3.tuple.Pair; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.dhp.schema.oaf.Qualifier; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMissingSubject extends UpdateMatcher> { public EnrichMissingSubject() { - super(true); + super(true, + pair -> Topic.fromPath("ENRICH/MISSING/SUBJECT/" + pair.getLeft()), + (p, pair) -> p.getSubjects().add(pair.getRight()), + pair -> pair.getLeft() + "::" + pair.getRight()); } @Override - protected List>> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List> findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { final Set existingTypes = target .getResult() .getSubject() @@ -40,20 +40,7 @@ public class EnrichMissingSubject extends UpdateMatcher> { .stream() .filter(pid -> !existingTypes.contains(pid.getQualifier().getClassid())) .map(ConversionUtils::oafSubjectToPair) - .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) .collect(Collectors.toList()); } - public UpdateInfo> generateUpdateInfo(final Pair highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - - return new UpdateInfo<>( - Topic.fromPath("ENRICH/MISSING/SUBJECT/" + highlightValue.getLeft()), - highlightValue, source, target, - (p, pair) -> p.getSubjects().add(pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight(), dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java index fc1112d73..3ee4c8bdd 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java @@ -10,20 +10,20 @@ import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMoreOpenAccess extends UpdateMatcher { public EnrichMoreOpenAccess() { - super(true); + super(true, + i -> Topic.ENRICH_MORE_OA_VERSION, + (p, i) -> p.getInstances().add(i), + Instance::getUrl); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { final Set urls = target .getResult() .getInstance() @@ -41,19 +41,7 @@ public class EnrichMoreOpenAccess extends UpdateMatcher { .map(ConversionUtils::oafInstanceToBrokerInstances) .flatMap(List::stream) .filter(i -> !urls.contains(i.getUrl())) - .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo(final Instance highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MORE_OA_VERSION, - highlightValue, source, target, - (p, i) -> p.getInstances().add(i), - Instance::getUrl, dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java index 7984cc521..694c9460a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java @@ -9,20 +9,20 @@ import eu.dnetlib.broker.objects.Pid; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMorePid extends UpdateMatcher { public EnrichMorePid() { - super(true); + super(true, + pid -> Topic.ENRICH_MORE_PID, + (p, pid) -> p.getPids().add(pid), + pid -> pid.getType() + "::" + pid.getValue()); } @Override - protected List> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { final Set existingPids = target .getResult() .getPid() @@ -36,19 +36,7 @@ public class EnrichMorePid extends UpdateMatcher { .stream() .filter(pid -> !existingPids.contains(pid.getQualifier().getClassid() + "::" + pid.getValue())) .map(ConversionUtils::oafPidToBrokerPid) - .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) .collect(Collectors.toList()); } - public UpdateInfo generateUpdateInfo(final Pid highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - return new UpdateInfo<>( - Topic.ENRICH_MORE_PID, - highlightValue, source, target, - (p, pid) -> p.getPids().add(pid), - pid -> pid.getType() + "::" + pid.getValue(), dedupConfig); - } - } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java index 1a522c745..7d0b1a65e 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java @@ -10,20 +10,20 @@ import org.apache.commons.lang3.tuple.Pair; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.pace.config.DedupConfig; public class EnrichMoreSubject extends UpdateMatcher> { public EnrichMoreSubject() { - super(true); + super(true, + pair -> Topic.fromPath("ENRICH/MORE/SUBJECT/" + pair.getLeft()), + (p, pair) -> p.getSubjects().add(pair.getRight()), + pair -> pair.getLeft() + "::" + pair.getRight()); } @Override - protected List>> findUpdates(final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { + protected List> findDifferences(final ResultWithRelations source, + final ResultWithRelations target) { final Set existingSubjects = target .getResult() .getSubject() @@ -37,20 +37,7 @@ public class EnrichMoreSubject extends UpdateMatcher> { .stream() .filter(pid -> !existingSubjects.contains(pid.getQualifier().getClassid() + "::" + pid.getValue())) .map(ConversionUtils::oafSubjectToPair) - .map(i -> generateUpdateInfo(i, source, target, dedupConfig)) .collect(Collectors.toList()); } - public UpdateInfo> generateUpdateInfo(final Pair highlightValue, - final ResultWithRelations source, - final ResultWithRelations target, - final DedupConfig dedupConfig) { - - return new UpdateInfo<>( - Topic.fromPath("ENRICH/MORE/SUBJECT/" + highlightValue.getLeft()), - highlightValue, source, target, - (p, pair) -> p.getSubjects().add(pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight(), dedupConfig); - } - } From 30ea1bda8852bc001a77f7615d1b6d3fdf380d6f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 12 Jun 2020 10:42:35 +0200 Subject: [PATCH 67/88] oozie workflow --- .../broker/oa/GenerateEventsApplication.java | 5 +- .../generate_all/oozie_app/config-default.xml | 18 ++++ .../oa/generate_all/oozie_app/workflow.xml | 99 +++++++++++++++++++ .../dhp/broker/oa/generate_broker_events.json | 26 +++++ 4 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_broker_events.json diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java index a09767192..4d40ba80d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java @@ -51,7 +51,7 @@ public class GenerateEventsApplication { IOUtils .toString( GenerateEventsApplication.class - .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/merge_claims_parameters.json"))); + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/generate_broker_events.json"))); parser.parseArgument(args); final Boolean isSparkSessionManaged = Optional @@ -149,7 +149,8 @@ public class GenerateEventsApplication { return r4; } - private static Dataset relatedEntities(final Dataset targets, final Dataset rels, + private static Dataset relatedEntities(final Dataset targets, + final Dataset rels, final Class clazz) { return rels .joinWith(targets, targets.col("id").equalTo(rels.col("target")), "inner") diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/config-default.xml b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/config-default.xml new file mode 100644 index 000000000..2e0ed9aee --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/config-default.xml @@ -0,0 +1,18 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/workflow.xml b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/workflow.xml new file mode 100644 index 000000000..da573ae9c --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_all/oozie_app/workflow.xml @@ -0,0 +1,99 @@ + + + + + graphInputPath + the path where the graph is stored + + + eventsOutputPath + the path where the the events will be stored + + + isLookupUrl + the address of the lookUp service + + + dedupConfProfId + the id of a valid Dedup Configuration Profile + + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + oozieActionShareLibForSpark2 + oozie action sharelib for spark 2.* + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + spark 2.* extra listeners classname + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + spark 2.* sql query execution listeners classname + + + spark2YarnHistoryServerAddress + spark 2.* yarn history server address + + + spark2EventLogDir + spark 2.* event log dir location + + + + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + eu.dnetlib.dhp.broker.oa.GenerateEventsApplication + --graphPath${graphInputPath} + --eventsPath${eventsOutputPath} + --isLookupUrl${isLookupUrl} + --dedupConfProfile${dedupConfProfId} + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_broker_events.json b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_broker_events.json new file mode 100644 index 000000000..6ab6d9a2d --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/resources/eu/dnetlib/dhp/broker/oa/generate_broker_events.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "g", + "paramLongName": "graphPath", + "paramDescription": "the path where there the graph is stored", + "paramRequired": true + }, + { + "paramName": "o", + "paramLongName": "eventsPath", + "paramDescription": "the path where the generated events will be stored", + "paramRequired": true + }, + { + "paramName": "lu", + "paramLongName": "isLookupUrl", + "paramDescription": "the address of the ISLookUpService", + "paramRequired": true + }, + { + "paramName": "d", + "paramLongName": "dedupConfProfile", + "paramDescription": "the id of a valid Dedup Configuration Profile", + "paramRequired": true + } +] From ba8a024af930df4dbdf75ceb2ef113dd71fcfa74 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 12 Jun 2020 10:45:11 +0200 Subject: [PATCH 68/88] avoid NPEs merging titles --- .../eu/dnetlib/dhp/schema/oaf/Result.java | 13 ++- .../oa/graph/clean/NormalizeEmptyFields.java | 88 ------------------- 2 files changed, 5 insertions(+), 96 deletions(-) delete mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java index 09e77a244..c9d0ac7c7 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java @@ -254,28 +254,25 @@ public class Result extends OafEntity implements Serializable { final StructuredProperty p = baseMainTitle; title = title.stream().filter(t -> t != p).collect(Collectors.toList()); } -// -// -// title.remove(baseMainTitle); } StructuredProperty newMainTitle = null; if (r.getTitle() != null) { newMainTitle = getMainTitle(r.getTitle()); - if (newMainTitle != null) { + if (newMainTitle != null && title != null) { final StructuredProperty p = newMainTitle; title = title.stream().filter(t -> t != p).collect(Collectors.toList()); } - - // r.getTitle().remove(newMainTitle); } - if (newMainTitle != null && compareTrust(this, r) < 0) + if (newMainTitle != null && compareTrust(this, r) < 0) { baseMainTitle = newMainTitle; + } title = mergeLists(title, r.getTitle()); - if (title != null && baseMainTitle != null) + if (title != null && baseMainTitle != null) { title.add(baseMainTitle); + } relevantdate = mergeLists(relevantdate, r.getRelevantdate()); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java deleted file mode 100644 index 77537801f..000000000 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/NormalizeEmptyFields.java +++ /dev/null @@ -1,88 +0,0 @@ - -package eu.dnetlib.dhp.oa.graph.clean; - -import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils; -import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; -import eu.dnetlib.dhp.schema.oaf.Oaf; -import eu.dnetlib.dhp.schema.oaf.Qualifier; -import org.apache.commons.lang3.StringUtils; -import org.apache.spark.api.java.function.MapFunction; - -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; - -public class NormalizeEmptyFields implements MapFunction { - - private VocabularyGroup vocabularies; - - public NormalizeEmptyFields(VocabularyGroup vocabularies) { - this.vocabularies = vocabularies; - } - - @Override - public T call(T value) throws Exception { - - doNormalize(value); - - return value; - } - - private void doNormalize(Object o) { - if (Objects.isNull(o)) { - return; - } - - if (o instanceof Iterable) { - for (Object oi : (Iterable) o) { - doNormalize(oi); - } - } else { - - Class clazz = o.getClass(); - - if (clazz.isPrimitive() - || o instanceof Integer - || o instanceof Double - || o instanceof Float - || o instanceof Long - || o instanceof Boolean - || o instanceof String) { - return; - } else { - try { - for (Field field : getAllFields(new LinkedList<>(), clazz)) { - field.setAccessible(true); - Object value = field.get(o); - if (value instanceof Qualifier && Objects.isNull(value)) { - field.set(o, OafMapperUtils.unknown("", "")); - } else if (value instanceof Field && Objects.isNull(value)) { - - } else { - doNormalize(value); - } - } - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new RuntimeException(e); - } - } - } - } - - private static List getAllFields(List fields, Class clazz) { - fields.addAll(Arrays.asList(clazz.getDeclaredFields())); - - final Class superclass = clazz.getSuperclass(); - if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { - getAllFields(fields, superclass); - } - - return fields; - } - - public VocabularyGroup getVocabularies() { - return vocabularies; - } -} From 97b1c4057c12d9624aadef55bbd13678ad9ba379 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 12 Jun 2020 10:45:18 +0200 Subject: [PATCH 69/88] WIP: graph cleaner implementation --- .../dhp/oa/graph/clean/CleaningRule.java | 90 +++++------ .../dhp/oa/graph/clean/OafNavigator.java | 147 ++++++++++++++++++ .../oa/graph/raw/common/VocabularyGroup.java | 5 +- .../dhp/oa/graph/clean/CleaningRuleTest.java | 7 + .../eu/dnetlib/dhp/oa/graph/clean/result.json | 22 +++ 5 files changed, 217 insertions(+), 54 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java index 51b930962..88fc61298 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java @@ -1,86 +1,70 @@ package eu.dnetlib.dhp.oa.graph.clean; -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; - +import com.google.common.collect.Maps; +import eu.dnetlib.dhp.schema.oaf.Field; +import eu.dnetlib.dhp.schema.oaf.StructuredProperty; +import org.apache.commons.lang3.StringUtils; import org.apache.spark.api.java.function.MapFunction; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Qualifier; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + public class CleaningRule implements MapFunction { private VocabularyGroup vocabularies; + private Map> mapping = Maps.newHashMap(); + + public CleaningRule(VocabularyGroup vocabularies) { this.vocabularies = vocabularies; + + mapping.put(Qualifier.class, o -> patchQualifier(o)); + mapping.put(StructuredProperty.class, o -> patchSp(o)); + mapping.put(Field.class, o -> patchStringField(o)); } @Override public T call(T value) throws Exception { - doClean(value); + OafNavigator.apply(value, mapping); return value; } - private void doClean(Object o) { - if (Objects.isNull(o)) { - return; - } - - if (o instanceof Iterable) { - for (Object oi : (Iterable) o) { - doClean(oi); - } - } else { - - Class clazz = o.getClass(); - - if (clazz.isPrimitive() - || o instanceof Integer - || o instanceof Double - || o instanceof Float - || o instanceof Long - || o instanceof Boolean - || o instanceof String) { - return; - } else { - try { - for (Field field : getAllFields(new LinkedList<>(), clazz)) { - field.setAccessible(true); - Object value = field.get(o); - if (value instanceof Qualifier) { - Qualifier q = (Qualifier) value; - if (vocabularies.vocabularyExists(q.getSchemeid())) { - field.set(o, vocabularies.lookup(q.getSchemeid(), q.getClassid())); - } - - } else { - doClean(value); - } - } - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new RuntimeException(e); - } - } + private Object patchQualifier(Object o) { + Qualifier q = (Qualifier) o; + if (vocabularies.vocabularyExists(q.getSchemeid())) { + return vocabularies.lookup(q.getSchemeid(), q.getClassid()); } + return o; } - private static List getAllFields(List fields, Class clazz) { - fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + private Object patchSp(Object o) { + StructuredProperty sp = (StructuredProperty) o; + if (StringUtils.isBlank(sp.getValue())) { + return null; + } + return o; + } - final Class superclass = clazz.getSuperclass(); - if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { - getAllFields(fields, superclass); + private Object patchStringField(Object o) { + Field f = (Field) o; + try { + if (StringUtils.isBlank((String) f.getValue())) { + return null; + } + } catch (ClassCastException e) { + // ignored on purpose } - return fields; + return o; } public VocabularyGroup getVocabularies() { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java index c329a3111..40facc110 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java @@ -1,4 +1,151 @@ + package eu.dnetlib.dhp.oa.graph.clean; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import eu.dnetlib.dhp.schema.oaf.Oaf; +import scala.Tuple2; + +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.beans.Introspector; +import java.beans.PropertyDescriptor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.util.*; +import java.util.function.Function; + public class OafNavigator { + + public static E apply(E oaf, Map> mapping) { + reflect(oaf, mapping); + return oaf; + } + + public static void reflect(Object o, Map> mapping) { + visit(o, mapping); + } + + public static void visit(final Object thingy, Map> mapping) { + + try { + final Class clazz = thingy.getClass(); + + if (!isPrimitive(thingy) && clazz.getPackage().equals(Oaf.class.getPackage())) { + + final BeanInfo beanInfo = Introspector.getBeanInfo(clazz); + + for (final PropertyDescriptor descriptor : beanInfo.getPropertyDescriptors()) { + try { + final Object value = descriptor.getReadMethod().invoke(thingy); + + if (value != null && !isPrimitive(value)) { + + System.out.println("VISITING " + descriptor.getName() + " " + descriptor.getPropertyType()); + + if (Iterable.class.isAssignableFrom(descriptor.getPropertyType())) { + for(Object vi : (Iterable) value) { + + visit(vi, mapping); + } + } else { + + if (mapping.keySet().contains(value.getClass())) { + final Object newValue = mapping.get(value.getClass()).apply(value); + System.out.println("PATCHING " + descriptor.getName()+ " " + descriptor.getPropertyType()); + System.out.println("OLD VALUE " + getObjectMapper().writeValueAsString(value)); + System.out.println("NEW VALUE " + getObjectMapper().writeValueAsString(newValue)); + descriptor.getWriteMethod().invoke(newValue); + } + + visit(value, mapping); + } + } + + } catch (final IllegalArgumentException e) { + // handle this please + } catch (final IllegalAccessException e) { + // and this also + } catch (final InvocationTargetException e) { + // and this, too + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + } + } + } catch (final IntrospectionException e) { + // do something sensible here + } + } + + private static ObjectMapper getObjectMapper() { + final ObjectMapper mapper = new ObjectMapper(); + return mapper; + } + + private static void navigate(Object o, Map> mapping) { + if (Objects.isNull(o) || isPrimitive(o)) { + return; + } else { + try { + for (Field field : getAllFields(o.getClass())) { + System.out.println(field.getName()); + field.setAccessible(true); + Object value = field.get(o); + + if (Objects.nonNull(value)) { + final Class fieldType = field.getType(); + if ((fieldType.isArray() && !fieldType.getComponentType().isPrimitive())) { + Object[] fs = (Object[]) value; + for (Object fi : fs) { + navigate(fi, mapping); + } + } if (Iterable.class.isAssignableFrom(fieldType)) { + Iterable fs = (Iterable) value; + for (Object fi : fs) { + navigate(fi, mapping); + } + } else { + if (mapping.keySet().contains(value.getClass())) { + System.out.println("PATCHING " + field.getName()); + field.set(o, mapping.get(value.getClass()).apply(value)); + } + } + } + } + + } catch (IllegalAccessException | IllegalArgumentException e) { + throw new RuntimeException(e); + } + } + } + + private static boolean isPrimitive(Object o) { + return o.getClass().isPrimitive() + || o instanceof Class + || o instanceof Integer + || o instanceof Double + || o instanceof Float + || o instanceof Long + || o instanceof Boolean + || o instanceof String + || o instanceof Byte; + } + + private static List getAllFields(Class clazz) { + return getAllFields(new LinkedList<>(), clazz); + } + + private static List getAllFields(List fields, Class clazz) { + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + + final Class superclass = clazz.getSuperclass(); + if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { + getAllFields(fields, superclass); + } + + return fields; + } + } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java index ec95ade00..447228bf9 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java @@ -105,7 +105,10 @@ public class VocabularyGroup implements Serializable { } public Qualifier getTermAsQualifier(final String vocId, final String id) { - return vocs.get(vocId.toLowerCase()).getTermAsQualifier(id); + if (vocabularyExists(vocId)) { + return vocs.get(vocId.toLowerCase()).getTermAsQualifier(id); + } + return OafMapperUtils.qualifier(id, id, "", ""); } public Qualifier getSynonymAsQualifier(final String vocId, final String syn) { diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java index 019285cc3..dcc98b200 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java @@ -7,6 +7,7 @@ import static org.mockito.Mockito.lenient; import java.io.IOException; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -15,6 +16,7 @@ import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.platform.commons.util.StringUtils; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -82,6 +84,11 @@ public class CleaningRuleTest { // TODO add more assertions to verity the cleaned values System.out.println(MAPPER.writeValueAsString(p_out)); + assertTrue( + p_out + .getPid() + .stream() + .allMatch(sp -> StringUtils.isNotBlank(sp.getValue()))); } private Stream getAuthorPidTypes(Publication pub) { diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json index 435b001b7..89ebe5af5 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json @@ -380,6 +380,28 @@ "schemename": "dnet:pid_types" }, "value": "10.1007/s109090161569x" + }, + { + "dataInfo": { + "deletedbyinference": false, + "inferenceprovenance": "", + "inferred": false, + "invisible": false, + "provenanceaction": { + "classid": "sysimport:crosswalk:datasetarchive", + "classname": "sysimport:crosswalk:datasetarchive", + "schemeid": "dnet:provenanceActions", + "schemename": "dnet:provenanceActions" + }, + "trust": "0.9" + }, + "qualifier": { + "classid": "doi", + "classname": "doi", + "schemeid": "dnet:pid_types", + "schemename": "dnet:pid_types" + }, + "value": "" } ], "relevantdate": [ From 3ade2631b3a17d176db0e59d24921265791e7e9b Mon Sep 17 00:00:00 2001 From: Alessia Bardi Date: Fri, 12 Jun 2020 10:52:12 +0200 Subject: [PATCH 70/88] Constants for new rels: citations and reviews --- .../java/eu/dnetlib/dhp/schema/common/ModelConstants.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java index e32dd10fa..6db382823 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java @@ -34,6 +34,13 @@ public class ModelConstants { public static final String IS_PART_OF = "IsPartOf"; public static final String HAS_PARTS = "HasParts"; public static final String RELATIONSHIP = "relationship"; + public static final String CITATION = "citation"; + public static final String CITES = "cites"; + public static final String IS_CITED_BY = "IsCitedBy"; + public static final String REVIEW = "review"; + public static final String REVIEWS = "reviews"; + public static final String IS_REVIEWED_BY = "IsReviewedBy"; + public static final String RESULT_PROJECT = "resultProject"; public static final String OUTCOME = "outcome"; From ed8879ed8bd5b503bb57def80932ce2992cd8ed2 Mon Sep 17 00:00:00 2001 From: Alessia Bardi Date: Fri, 12 Jun 2020 10:55:56 +0200 Subject: [PATCH 71/88] deprecate PUBLICATION_DATASET --- .../java/eu/dnetlib/dhp/schema/common/ModelConstants.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java index 6db382823..34fd67e92 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java @@ -25,6 +25,11 @@ public class ModelConstants { public static final String ORP_RESULTTYPE_CLASSID = "other"; public static final String RESULT_RESULT = "resultResult"; + /** + * @deprecated + * Use {@link ModelConstants#RELATIONSHIP} instead. + */ + @Deprecated public static final String PUBLICATION_DATASET = "publicationDataset"; public static final String IS_RELATED_TO = "isRelatedTo"; public static final String SUPPLEMENT = "supplement"; From b347499745e1b418566e9de84eacbc49cae40991 Mon Sep 17 00:00:00 2001 From: Alessia Bardi Date: Fri, 12 Jun 2020 10:58:02 +0200 Subject: [PATCH 72/88] do not use deprecated subreltype --- .../eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index e44f830df..7b0967a6b 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -4,13 +4,7 @@ package eu.dnetlib.dhp.oa.graph.raw; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.createOpenaireId; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.field; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_ACCESS_MODES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_LANGUAGES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PID_TYPES; -import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_PUBLICATION_RESOURCE; -import static eu.dnetlib.dhp.schema.common.ModelConstants.IS_RELATED_TO; -import static eu.dnetlib.dhp.schema.common.ModelConstants.PUBLICATION_DATASET; -import static eu.dnetlib.dhp.schema.common.ModelConstants.RESULT_RESULT; +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import java.util.ArrayList; import java.util.List; @@ -281,12 +275,12 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { res .add( getRelation( - docId, otherId, RESULT_RESULT, PUBLICATION_DATASET, IS_RELATED_TO, collectedFrom, info, + docId, otherId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, collectedFrom, info, lastUpdateTimestamp)); res .add( getRelation( - otherId, docId, RESULT_RESULT, PUBLICATION_DATASET, IS_RELATED_TO, collectedFrom, info, + otherId, docId, RESULT_RESULT, RELATIONSHIP, IS_RELATED_TO, collectedFrom, info, lastUpdateTimestamp)); } } From cdb1956fe9cade7202c69733a264a2fb55b7f297 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 12 Jun 2020 11:36:59 +0200 Subject: [PATCH 73/88] WIP: graph cleaner implementation --- .../dhp/oa/graph/clean/OafNavigator2.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java new file mode 100644 index 000000000..74f064e47 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java @@ -0,0 +1,94 @@ + +package eu.dnetlib.dhp.oa.graph.clean; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.schema.oaf.Oaf; + +import java.lang.reflect.Field; +import java.util.*; +import java.util.function.Function; + +public class OafNavigator2 { + + public static E apply(E oaf, Map> mapping) { + navigate(oaf, mapping); + return oaf; + } + + private static void navigate(Object o, Map> mapping) { + if (Objects.isNull(o) || isPrimitive(o)) { + return; + } else { + try { + for (Field field : getAllFields(o.getClass())) { + System.out.println("VISITING " + field.getName() + " in " + o.getClass()); + field.setAccessible(true); + Object value = field.get(o); + + if (Objects.nonNull(value)) { + final Class fieldType = field.getType(); + if ((fieldType.isArray() && !fieldType.getComponentType().isPrimitive())) { + Object[] fs = (Object[]) value; + for (Object fi : fs) { + navigate(fi, mapping); + } + } if (Iterable.class.isAssignableFrom(fieldType)) { + Iterable fs = (Iterable) value; + for (Object fi : fs) { + navigate(fi, mapping); + } + } else { + final Function cleaningFn = mapping.get(value.getClass()); + if (Objects.nonNull(cleaningFn)) { + final Object newValue = cleaningFn.apply(value); + if (!Objects.equals(value, newValue)) { + System.out.println("PATCHING " + field.getName()+ " " + value.getClass()); + System.out.println("OLD VALUE " + getObjectMapper().writeValueAsString(value)); + System.out.println("NEW VALUE " + getObjectMapper().writeValueAsString(newValue)); + field.set(o, newValue); + } + } + } + } + } + + } catch (IllegalAccessException | IllegalArgumentException | JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + + private static ObjectMapper getObjectMapper() { + final ObjectMapper mapper = new ObjectMapper(); + return mapper; + } + + private static boolean isPrimitive(Object o) { + return o.getClass().isPrimitive() + || o instanceof Class + || o instanceof Integer + || o instanceof Double + || o instanceof Float + || o instanceof Long + || o instanceof Boolean + || o instanceof String + || o instanceof Byte; + } + + private static List getAllFields(Class clazz) { + return getAllFields(new LinkedList<>(), clazz); + } + + private static List getAllFields(List fields, Class clazz) { + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + + final Class superclass = clazz.getSuperclass(); + if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { + getAllFields(fields, superclass); + } + + return fields; + } + +} From 463489f59f68c903874593ebd0cea167e50ad5d0 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 12 Jun 2020 12:03:25 +0200 Subject: [PATCH 74/88] code formatting --- .../eu/dnetlib/dhp/common/PacePerson.java | 35 +++++++++---------- .../eu/dnetlib/dhp/common/PacePersonTest.java | 33 ++++++++--------- .../dhp/schema/common/ModelConstants.java | 4 +-- .../dhp/oa/graph/clean/CleaningRule.java | 24 ++++++------- .../dhp/oa/graph/clean/OafNavigator.java | 22 +++++++----- .../dhp/oa/graph/clean/OafNavigator2.java | 22 ++++++------ .../raw/GenerateEntitiesApplication.java | 2 -- .../dhp/oa/graph/clean/CleaningRuleTest.java | 8 ++--- .../oa/provision/utils/XmlRecordFactory.java | 5 ++- 9 files changed, 79 insertions(+), 76 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java index ccd42225a..6e02ca614 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java @@ -17,12 +17,11 @@ import com.google.common.collect.Lists; import com.google.common.hash.Hashing; /** -* PacePerson tries to derive information from the fullname string of an author. -* Such informations are Names, Surnames an Fullname split into terms. It provides also an additional field for -* the original data. -* The calculation of the names and the surnames is not always possible. When it is impossible to assert which are the -* names and the surnames, the lists are empty. -* */ + * PacePerson tries to derive information from the fullname string of an author. Such informations are Names, Surnames + * an Fullname split into terms. It provides also an additional field for the original data. The calculation of the + * names and the surnames is not always possible. When it is impossible to assert which are the names and the surnames, + * the lists are empty. + */ public class PacePerson { private static final String UTF8 = "UTF-8"; @@ -34,18 +33,18 @@ public class PacePerson { private static Set particles = null; /** - * Capitalizes a string + * Capitalizes a string * * @param s the string to capitalize * @return the input string with capital letter - * */ + */ public static final String capitalize(final String s) { return WordUtils.capitalize(s.toLowerCase(), ' ', '-'); } /** - * Adds a dot to a string with length equals to 1 - * */ + * Adds a dot to a string with length equals to 1 + */ public static final String dotAbbreviations(final String s) { return s.length() == 1 ? s + "." : s; } @@ -63,11 +62,11 @@ public class PacePerson { } /** - * The constructor of the class. It fills the fields of the class basing on the input fullname. + * The constructor of the class. It fills the fields of the class basing on the input fullname. * * @param s the input string (fullname of the author) * @param aggressive set the string normalization type - * */ + */ public PacePerson(String s, final boolean aggressive) { original = s; s = Normalizer.normalize(s, Normalizer.Form.NFD); @@ -86,7 +85,7 @@ public class PacePerson { // s = s.replaceAll("[\\W&&[^,-]]", ""); } - //if the string contains a comma, it can derive surname and name by splitting on it + // if the string contains a comma, it can derive surname and name by splitting on it if (s.contains(",")) { final String[] arr = s.split(","); if (arr.length == 1) { @@ -97,23 +96,23 @@ public class PacePerson { fullname.addAll(surname); fullname.addAll(name); } - } else { //otherwise, it should rely on CAPS terms and short terms + } else { // otherwise, it should rely on CAPS terms and short terms fullname = splitTerms(s); int lastInitialPosition = fullname.size(); boolean hasSurnameInUpperCase = false; - //computes lastInitialPosition and hasSurnameInUpperCase + // computes lastInitialPosition and hasSurnameInUpperCase for (int i = 0; i < fullname.size(); i++) { final String term = fullname.get(i); if (term.length() == 1) { - lastInitialPosition = i; //first word in the name longer than 1 (to avoid name with dots) + lastInitialPosition = i; // first word in the name longer than 1 (to avoid name with dots) } else if (term.equals(term.toUpperCase())) { - hasSurnameInUpperCase = true; //if one of the words is CAPS + hasSurnameInUpperCase = true; // if one of the words is CAPS } } - //manages particular cases of fullnames + // manages particular cases of fullnames if (lastInitialPosition < fullname.size() - 1) { // Case: Michele G. Artini name = fullname.subList(0, lastInitialPosition + 1); surname = fullname.subList(lastInitialPosition + 1, fullname.size()); diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java index 7ee60a0aa..5ebd7213e 100644 --- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/PacePersonTest.java @@ -1,26 +1,27 @@ -package eu.dnetlib.dhp.common; -import org.junit.jupiter.api.Test; +package eu.dnetlib.dhp.common; import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; + public class PacePersonTest { - @Test - public void pacePersonTest1(){ + @Test + public void pacePersonTest1() { - PacePerson p = new PacePerson("Artini, Michele", false); - assertEquals("Artini",p.getSurnameString()); - assertEquals("Michele", p.getNameString()); - assertEquals("Artini, Michele", p.getNormalisedFullname()); - } + PacePerson p = new PacePerson("Artini, Michele", false); + assertEquals("Artini", p.getSurnameString()); + assertEquals("Michele", p.getNameString()); + assertEquals("Artini, Michele", p.getNormalisedFullname()); + } - @Test - public void pacePersonTest2(){ - PacePerson p = new PacePerson("Michele G. Artini", false); - assertEquals("Artini, Michele G.", p.getNormalisedFullname()); - assertEquals("Michele G", p.getNameString()); - assertEquals("Artini", p.getSurnameString()); - } + @Test + public void pacePersonTest2() { + PacePerson p = new PacePerson("Michele G. Artini", false); + assertEquals("Artini, Michele G.", p.getNormalisedFullname()); + assertEquals("Michele G", p.getNameString()); + assertEquals("Artini", p.getSurnameString()); + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java index a22132d4c..a3c1610db 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java @@ -27,8 +27,7 @@ public class ModelConstants { public static final String RESULT_RESULT = "resultResult"; /** - * @deprecated - * Use {@link ModelConstants#RELATIONSHIP} instead. + * @deprecated Use {@link ModelConstants#RELATIONSHIP} instead. */ @Deprecated public static final String PUBLICATION_DATASET = "publicationDataset"; @@ -47,7 +46,6 @@ public class ModelConstants { public static final String REVIEWS = "reviews"; public static final String IS_REVIEWED_BY = "IsReviewedBy"; - public static final String RESULT_PROJECT = "resultProject"; public static final String OUTCOME = "outcome"; public static final String IS_PRODUCED_BY = "isProducedBy"; diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java index 88fc61298..a7efbb16a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java @@ -1,27 +1,27 @@ package eu.dnetlib.dhp.oa.graph.clean; -import com.google.common.collect.Maps; -import eu.dnetlib.dhp.schema.oaf.Field; -import eu.dnetlib.dhp.schema.oaf.StructuredProperty; -import org.apache.commons.lang3.StringUtils; -import org.apache.spark.api.java.function.MapFunction; - -import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; -import eu.dnetlib.dhp.schema.oaf.Oaf; -import eu.dnetlib.dhp.schema.oaf.Qualifier; - import java.util.Map; import java.util.Objects; import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.api.java.function.MapFunction; + +import com.google.common.collect.Maps; + +import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; +import eu.dnetlib.dhp.schema.oaf.Field; +import eu.dnetlib.dhp.schema.oaf.Oaf; +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import eu.dnetlib.dhp.schema.oaf.StructuredProperty; + public class CleaningRule implements MapFunction { private VocabularyGroup vocabularies; private Map> mapping = Maps.newHashMap(); - public CleaningRule(VocabularyGroup vocabularies) { this.vocabularies = vocabularies; @@ -33,7 +33,7 @@ public class CleaningRule implements MapFunction { @Override public T call(T value) throws Exception { - OafNavigator.apply(value, mapping); + OafNavigator2.apply(value, mapping); return value; } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java index 40facc110..8b27dabf6 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java @@ -1,12 +1,6 @@ package eu.dnetlib.dhp.oa.graph.clean; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import eu.dnetlib.dhp.schema.oaf.Oaf; -import scala.Tuple2; - import java.beans.BeanInfo; import java.beans.IntrospectionException; import java.beans.Introspector; @@ -16,6 +10,13 @@ import java.lang.reflect.InvocationTargetException; import java.util.*; import java.util.function.Function; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; + +import eu.dnetlib.dhp.schema.oaf.Oaf; +import scala.Tuple2; + public class OafNavigator { public static E apply(E oaf, Map> mapping) { @@ -45,7 +46,7 @@ public class OafNavigator { System.out.println("VISITING " + descriptor.getName() + " " + descriptor.getPropertyType()); if (Iterable.class.isAssignableFrom(descriptor.getPropertyType())) { - for(Object vi : (Iterable) value) { + for (Object vi : (Iterable) value) { visit(vi, mapping); } @@ -53,7 +54,9 @@ public class OafNavigator { if (mapping.keySet().contains(value.getClass())) { final Object newValue = mapping.get(value.getClass()).apply(value); - System.out.println("PATCHING " + descriptor.getName()+ " " + descriptor.getPropertyType()); + System.out + .println( + "PATCHING " + descriptor.getName() + " " + descriptor.getPropertyType()); System.out.println("OLD VALUE " + getObjectMapper().writeValueAsString(value)); System.out.println("NEW VALUE " + getObjectMapper().writeValueAsString(newValue)); descriptor.getWriteMethod().invoke(newValue); @@ -101,7 +104,8 @@ public class OafNavigator { for (Object fi : fs) { navigate(fi, mapping); } - } if (Iterable.class.isAssignableFrom(fieldType)) { + } + if (Iterable.class.isAssignableFrom(fieldType)) { Iterable fs = (Iterable) value; for (Object fi : fs) { navigate(fi, mapping); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java index 74f064e47..264c95daf 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java @@ -1,14 +1,15 @@ package eu.dnetlib.dhp.oa.graph.clean; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.dhp.schema.oaf.Oaf; - import java.lang.reflect.Field; import java.util.*; import java.util.function.Function; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.schema.oaf.Oaf; + public class OafNavigator2 { public static E apply(E oaf, Map> mapping) { @@ -22,7 +23,7 @@ public class OafNavigator2 { } else { try { for (Field field : getAllFields(o.getClass())) { - System.out.println("VISITING " + field.getName() + " in " + o.getClass()); + //System.out.println("VISITING " + field.getName() + " in " + o.getClass()); field.setAccessible(true); Object value = field.get(o); @@ -33,7 +34,8 @@ public class OafNavigator2 { for (Object fi : fs) { navigate(fi, mapping); } - } if (Iterable.class.isAssignableFrom(fieldType)) { + } + if (Iterable.class.isAssignableFrom(fieldType)) { Iterable fs = (Iterable) value; for (Object fi : fs) { navigate(fi, mapping); @@ -43,9 +45,9 @@ public class OafNavigator2 { if (Objects.nonNull(cleaningFn)) { final Object newValue = cleaningFn.apply(value); if (!Objects.equals(value, newValue)) { - System.out.println("PATCHING " + field.getName()+ " " + value.getClass()); - System.out.println("OLD VALUE " + getObjectMapper().writeValueAsString(value)); - System.out.println("NEW VALUE " + getObjectMapper().writeValueAsString(newValue)); + //System.out.println("PATCHING " + field.getName() + " " + value.getClass()); + //System.out.println("OLD VALUE " + getObjectMapper().writeValueAsString(value)); + //System.out.println("NEW VALUE " + getObjectMapper().writeValueAsString(newValue)); field.set(o, newValue); } } @@ -53,7 +55,7 @@ public class OafNavigator2 { } } - } catch (IllegalAccessException | IllegalArgumentException | JsonProcessingException e) { + } catch (IllegalAccessException | IllegalArgumentException /*| JsonProcessingException*/ e) { throw new RuntimeException(e); } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java index 3411dc898..3568dc52a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java @@ -140,11 +140,9 @@ public class GenerateEntitiesApplication { final String type = StringUtils.substringAfter(id, ":"); switch (type.toLowerCase()) { - case "oaf-store-claim": case "oaf-store-cleaned": case "oaf-store-claim": return new OafToOafMapper(vocs, false).processMdRecord(s); - case "odf-store-claim": case "odf-store-cleaned": case "odf-store-claim": return new OdfToOafMapper(vocs, false).processMdRecord(s); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java index dcc98b200..7792f64c6 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java @@ -84,11 +84,9 @@ public class CleaningRuleTest { // TODO add more assertions to verity the cleaned values System.out.println(MAPPER.writeValueAsString(p_out)); - assertTrue( - p_out - .getPid() - .stream() - .allMatch(sp -> StringUtils.isNotBlank(sp.getValue()))); + /* + * assertTrue( p_out .getPid() .stream() .allMatch(sp -> StringUtils.isNotBlank(sp.getValue()))); + */ } private Stream getAuthorPidTypes(Publication pub) { diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java index 1e9c49920..b2aa01dc7 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java @@ -897,7 +897,10 @@ public class XmlRecordFactory implements Serializable { metadata.add(XmlSerializationUtils.mapQualifier("contracttype", p.getContracttype())); } if (p.getOamandatepublications() != null) { - metadata.add(XmlSerializationUtils.asXmlElement("oamandatepublications", p.getOamandatepublications().getValue())); + metadata + .add( + XmlSerializationUtils + .asXmlElement("oamandatepublications", p.getOamandatepublications().getValue())); } if (p.getEcsc39() != null) { metadata.add(XmlSerializationUtils.asXmlElement("ecsc39", p.getEcsc39().getValue())); From f0746a76055527b5b4855af4533bebe76f4e7c30 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 12 Jun 2020 12:21:03 +0200 Subject: [PATCH 75/88] [maven-release-plugin] prepare release dhp-1.2.2 --- dhp-build/dhp-build-assembly-resources/pom.xml | 2 +- dhp-build/dhp-build-properties-maven-plugin/pom.xml | 2 +- dhp-build/dhp-code-style/pom.xml | 2 +- dhp-build/pom.xml | 2 +- dhp-common/pom.xml | 2 +- dhp-schemas/pom.xml | 2 +- dhp-workflows/dhp-actionmanager/pom.xml | 2 +- dhp-workflows/dhp-aggregation/pom.xml | 2 +- dhp-workflows/dhp-blacklist/pom.xml | 2 +- dhp-workflows/dhp-broker-events/pom.xml | 6 ++---- dhp-workflows/dhp-dedup-openaire/pom.xml | 2 +- dhp-workflows/dhp-dedup-scholexplorer/pom.xml | 2 +- dhp-workflows/dhp-distcp/pom.xml | 2 +- dhp-workflows/dhp-doiboost/pom.xml | 5 ++--- dhp-workflows/dhp-enrichment/pom.xml | 2 +- dhp-workflows/dhp-graph-mapper/pom.xml | 2 +- dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml | 2 +- dhp-workflows/dhp-graph-provision/pom.xml | 2 +- dhp-workflows/dhp-stats-update/pom.xml | 2 +- dhp-workflows/dhp-worfklow-profiles/pom.xml | 2 +- dhp-workflows/pom.xml | 2 +- pom.xml | 4 ++-- 22 files changed, 25 insertions(+), 28 deletions(-) diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index 8bae191d3..bcad5df46 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.2-SNAPSHOT + 1.2.2 dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index ad8cd57b4..21c1d93b3 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.2-SNAPSHOT + 1.2.2 dhp-build-properties-maven-plugin diff --git a/dhp-build/dhp-code-style/pom.xml b/dhp-build/dhp-code-style/pom.xml index 08f5de9ee..0e2ed45d8 100644 --- a/dhp-build/dhp-code-style/pom.xml +++ b/dhp-build/dhp-code-style/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp-code-style - 1.2.2-SNAPSHOT + 1.2.2 jar diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index 369e25b24..f732962b4 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.2 dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 60e66f45a..bc4ac7bbb 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.2 ../ diff --git a/dhp-schemas/pom.xml b/dhp-schemas/pom.xml index 5e864cf94..92b8a60a9 100644 --- a/dhp-schemas/pom.xml +++ b/dhp-schemas/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.2 ../ diff --git a/dhp-workflows/dhp-actionmanager/pom.xml b/dhp-workflows/dhp-actionmanager/pom.xml index ec6247102..021efc9b4 100644 --- a/dhp-workflows/dhp-actionmanager/pom.xml +++ b/dhp-workflows/dhp-actionmanager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.2-SNAPSHOT + 1.2.2 dhp-actionmanager diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index 9f082df70..bd95378db 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.2-SNAPSHOT + 1.2.2 dhp-aggregation diff --git a/dhp-workflows/dhp-blacklist/pom.xml b/dhp-workflows/dhp-blacklist/pom.xml index a3cc15b74..c187b803a 100644 --- a/dhp-workflows/dhp-blacklist/pom.xml +++ b/dhp-workflows/dhp-blacklist/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-broker-events/pom.xml b/dhp-workflows/dhp-broker-events/pom.xml index cc323d109..41af717b8 100644 --- a/dhp-workflows/dhp-broker-events/pom.xml +++ b/dhp-workflows/dhp-broker-events/pom.xml @@ -1,11 +1,9 @@ - + dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index 44cf9e67c..5989d8bee 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml index 429c8a648..995ceaebb 100644 --- a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-distcp/pom.xml b/dhp-workflows/dhp-distcp/pom.xml index 8454c29a4..029f6c90d 100644 --- a/dhp-workflows/dhp-distcp/pom.xml +++ b/dhp-workflows/dhp-distcp/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-doiboost/pom.xml b/dhp-workflows/dhp-doiboost/pom.xml index 168442942..d9a3f505e 100644 --- a/dhp-workflows/dhp-doiboost/pom.xml +++ b/dhp-workflows/dhp-doiboost/pom.xml @@ -1,10 +1,9 @@ - + dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-enrichment/pom.xml b/dhp-workflows/dhp-enrichment/pom.xml index 2dc0f2436..20eac9b6b 100644 --- a/dhp-workflows/dhp-enrichment/pom.xml +++ b/dhp-workflows/dhp-enrichment/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index aee3d27c1..8db544998 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml b/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml index e0ce739cf..4f5527a5e 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index 62bf7186c..e742ff7dc 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/dhp-stats-update/pom.xml b/dhp-workflows/dhp-stats-update/pom.xml index d6ec4e6ab..c7ef23aec 100644 --- a/dhp-workflows/dhp-stats-update/pom.xml +++ b/dhp-workflows/dhp-stats-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 dhp-stats-update diff --git a/dhp-workflows/dhp-worfklow-profiles/pom.xml b/dhp-workflows/dhp-worfklow-profiles/pom.xml index cb20db57e..e45b3b161 100644 --- a/dhp-workflows/dhp-worfklow-profiles/pom.xml +++ b/dhp-workflows/dhp-worfklow-profiles/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2-SNAPSHOT + 1.2.2 4.0.0 diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 11e5be0b6..674e32a48 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.2 ../ diff --git a/pom.xml b/pom.xml index e0ee18900..72423753a 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.dnetlib.dhp dhp - 1.2.2-SNAPSHOT + 1.2.2 pom @@ -38,7 +38,7 @@ scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git https://code-repo.d4science.org/D-Net/dnet-hadoop/ - HEAD + dhp-1.2.2 This module is the root descriptor for the dnet-hadoop project From c4d9f1837f8e6a9e6ee4d588f847e78a07124a77 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 12 Jun 2020 12:21:08 +0200 Subject: [PATCH 76/88] [maven-release-plugin] prepare for next development iteration --- dhp-build/dhp-build-assembly-resources/pom.xml | 2 +- dhp-build/dhp-build-properties-maven-plugin/pom.xml | 2 +- dhp-build/dhp-code-style/pom.xml | 2 +- dhp-build/pom.xml | 2 +- dhp-common/pom.xml | 2 +- dhp-schemas/pom.xml | 2 +- dhp-workflows/dhp-actionmanager/pom.xml | 2 +- dhp-workflows/dhp-aggregation/pom.xml | 2 +- dhp-workflows/dhp-blacklist/pom.xml | 2 +- dhp-workflows/dhp-broker-events/pom.xml | 2 +- dhp-workflows/dhp-dedup-openaire/pom.xml | 2 +- dhp-workflows/dhp-dedup-scholexplorer/pom.xml | 2 +- dhp-workflows/dhp-distcp/pom.xml | 2 +- dhp-workflows/dhp-doiboost/pom.xml | 2 +- dhp-workflows/dhp-enrichment/pom.xml | 2 +- dhp-workflows/dhp-graph-mapper/pom.xml | 2 +- dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml | 2 +- dhp-workflows/dhp-graph-provision/pom.xml | 2 +- dhp-workflows/dhp-stats-update/pom.xml | 2 +- dhp-workflows/dhp-worfklow-profiles/pom.xml | 2 +- dhp-workflows/pom.xml | 2 +- pom.xml | 4 ++-- 22 files changed, 23 insertions(+), 23 deletions(-) diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index bcad5df46..2a89a26fd 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.2 + 1.2.3-SNAPSHOT dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index 21c1d93b3..5be114e3c 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.2 + 1.2.3-SNAPSHOT dhp-build-properties-maven-plugin diff --git a/dhp-build/dhp-code-style/pom.xml b/dhp-build/dhp-code-style/pom.xml index 0e2ed45d8..515ed35ce 100644 --- a/dhp-build/dhp-code-style/pom.xml +++ b/dhp-build/dhp-code-style/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp-code-style - 1.2.2 + 1.2.3-SNAPSHOT jar diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index f732962b4..d2dcbc36e 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.2.2 + 1.2.3-SNAPSHOT dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index bc4ac7bbb..0e7652dd3 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.2 + 1.2.3-SNAPSHOT ../ diff --git a/dhp-schemas/pom.xml b/dhp-schemas/pom.xml index 92b8a60a9..56fb8ead2 100644 --- a/dhp-schemas/pom.xml +++ b/dhp-schemas/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.2 + 1.2.3-SNAPSHOT ../ diff --git a/dhp-workflows/dhp-actionmanager/pom.xml b/dhp-workflows/dhp-actionmanager/pom.xml index 021efc9b4..b50c6705b 100644 --- a/dhp-workflows/dhp-actionmanager/pom.xml +++ b/dhp-workflows/dhp-actionmanager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.2 + 1.2.3-SNAPSHOT dhp-actionmanager diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index bd95378db..fdb4467d6 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.2 + 1.2.3-SNAPSHOT dhp-aggregation diff --git a/dhp-workflows/dhp-blacklist/pom.xml b/dhp-workflows/dhp-blacklist/pom.xml index c187b803a..04d334cd7 100644 --- a/dhp-workflows/dhp-blacklist/pom.xml +++ b/dhp-workflows/dhp-blacklist/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-broker-events/pom.xml b/dhp-workflows/dhp-broker-events/pom.xml index 41af717b8..f060a60bc 100644 --- a/dhp-workflows/dhp-broker-events/pom.xml +++ b/dhp-workflows/dhp-broker-events/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index 5989d8bee..1f5f2620e 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml index 995ceaebb..e9e11b417 100644 --- a/dhp-workflows/dhp-dedup-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-dedup-scholexplorer/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-distcp/pom.xml b/dhp-workflows/dhp-distcp/pom.xml index 029f6c90d..5707ddfc5 100644 --- a/dhp-workflows/dhp-distcp/pom.xml +++ b/dhp-workflows/dhp-distcp/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-doiboost/pom.xml b/dhp-workflows/dhp-doiboost/pom.xml index d9a3f505e..39bb81ec1 100644 --- a/dhp-workflows/dhp-doiboost/pom.xml +++ b/dhp-workflows/dhp-doiboost/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-enrichment/pom.xml b/dhp-workflows/dhp-enrichment/pom.xml index 20eac9b6b..e71a72f3e 100644 --- a/dhp-workflows/dhp-enrichment/pom.xml +++ b/dhp-workflows/dhp-enrichment/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index 8db544998..f650f1c17 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml b/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml index 4f5527a5e..2466ca8e2 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index e742ff7dc..b0aec1e5d 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-stats-update/pom.xml b/dhp-workflows/dhp-stats-update/pom.xml index c7ef23aec..397bd8d08 100644 --- a/dhp-workflows/dhp-stats-update/pom.xml +++ b/dhp-workflows/dhp-stats-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 dhp-stats-update diff --git a/dhp-workflows/dhp-worfklow-profiles/pom.xml b/dhp-workflows/dhp-worfklow-profiles/pom.xml index e45b3b161..e03362034 100644 --- a/dhp-workflows/dhp-worfklow-profiles/pom.xml +++ b/dhp-workflows/dhp-worfklow-profiles/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.2 + 1.2.3-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 674e32a48..8d8d57c84 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.2.2 + 1.2.3-SNAPSHOT ../ diff --git a/pom.xml b/pom.xml index 72423753a..06e2b7aaf 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.dnetlib.dhp dhp - 1.2.2 + 1.2.3-SNAPSHOT pom @@ -38,7 +38,7 @@ scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git https://code-repo.d4science.org/D-Net/dnet-hadoop/ - dhp-1.2.2 + HEAD This module is the root descriptor for the dnet-hadoop project From bed65a1be68129d7e9ddc25675ca53d5b80425ce Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 12 Jun 2020 18:25:47 +0200 Subject: [PATCH 77/88] WIP: graph cleaner implementation --- .../dhp/oa/graph/clean/CleaningRule.java | 62 ++++---- .../dhp/oa/graph/clean/OafNavigator.java | 140 ++++-------------- .../dhp/oa/graph/clean/OafNavigator2.java | 96 ------------ 3 files changed, 58 insertions(+), 240 deletions(-) delete mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java index a7efbb16a..6d7a262be 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java @@ -2,16 +2,13 @@ package eu.dnetlib.dhp.oa.graph.clean; import java.util.Map; -import java.util.Objects; -import java.util.function.Function; +import java.util.function.Consumer; -import org.apache.commons.lang3.StringUtils; import org.apache.spark.api.java.function.MapFunction; import com.google.common.collect.Maps; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; -import eu.dnetlib.dhp.schema.oaf.Field; import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Qualifier; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; @@ -20,54 +17,47 @@ public class CleaningRule implements MapFunction { private VocabularyGroup vocabularies; - private Map> mapping = Maps.newHashMap(); + private Map> mapping = Maps.newHashMap(); public CleaningRule(VocabularyGroup vocabularies) { this.vocabularies = vocabularies; - - mapping.put(Qualifier.class, o -> patchQualifier(o)); - mapping.put(StructuredProperty.class, o -> patchSp(o)); - mapping.put(Field.class, o -> patchStringField(o)); + setMappings(vocabularies); } @Override public T call(T value) throws Exception { - OafNavigator2.apply(value, mapping); + OafNavigator.apply(value, mapping); return value; } - private Object patchQualifier(Object o) { - Qualifier q = (Qualifier) o; - if (vocabularies.vocabularyExists(q.getSchemeid())) { - return vocabularies.lookup(q.getSchemeid(), q.getClassid()); - } - return o; - } - - private Object patchSp(Object o) { - StructuredProperty sp = (StructuredProperty) o; - if (StringUtils.isBlank(sp.getValue())) { - return null; - } - return o; - } - - private Object patchStringField(Object o) { - Field f = (Field) o; - try { - if (StringUtils.isBlank((String) f.getValue())) { - return null; + /** + * Populates the mapping for the Oaf types subject to cleaning + * + * @param vocabularies + */ + private void setMappings(VocabularyGroup vocabularies) { + mapping.put(Qualifier.class, o -> { + Qualifier q = (Qualifier) o; + if (vocabularies.vocabularyExists(q.getSchemeid())) { + Qualifier newValue = vocabularies.lookup(q.getSchemeid(), q.getClassid()); + q.setClassid(newValue.getClassid()); + q.setClassname(newValue.getClassname()); } - } catch (ClassCastException e) { - // ignored on purpose - } - - return o; + }); + mapping.put(StructuredProperty.class, o -> { + StructuredProperty sp = (StructuredProperty) o; + // TODO implement a policy + /* + * if (StringUtils.isBlank(sp.getValue())) { sp.setValue(null); sp.setQualifier(null); sp.setDataInfo(null); + * } + */ + }); } public VocabularyGroup getVocabularies() { return vocabularies; } + } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java index 8b27dabf6..2cc499577 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java @@ -1,132 +1,56 @@ package eu.dnetlib.dhp.oa.graph.clean; -import java.beans.BeanInfo; -import java.beans.IntrospectionException; -import java.beans.Introspector; -import java.beans.PropertyDescriptor; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.util.*; -import java.util.function.Function; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; +import java.util.function.Consumer; import eu.dnetlib.dhp.schema.oaf.Oaf; -import scala.Tuple2; public class OafNavigator { - public static E apply(E oaf, Map> mapping) { - reflect(oaf, mapping); + public static E apply(E oaf, Map> mapping) { + try { + navigate(oaf, mapping); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } return oaf; } - public static void reflect(Object o, Map> mapping) { - visit(o, mapping); - } - - public static void visit(final Object thingy, Map> mapping) { - - try { - final Class clazz = thingy.getClass(); - - if (!isPrimitive(thingy) && clazz.getPackage().equals(Oaf.class.getPackage())) { - - final BeanInfo beanInfo = Introspector.getBeanInfo(clazz); - - for (final PropertyDescriptor descriptor : beanInfo.getPropertyDescriptors()) { - try { - final Object value = descriptor.getReadMethod().invoke(thingy); - - if (value != null && !isPrimitive(value)) { - - System.out.println("VISITING " + descriptor.getName() + " " + descriptor.getPropertyType()); - - if (Iterable.class.isAssignableFrom(descriptor.getPropertyType())) { - for (Object vi : (Iterable) value) { - - visit(vi, mapping); - } - } else { - - if (mapping.keySet().contains(value.getClass())) { - final Object newValue = mapping.get(value.getClass()).apply(value); - System.out - .println( - "PATCHING " + descriptor.getName() + " " + descriptor.getPropertyType()); - System.out.println("OLD VALUE " + getObjectMapper().writeValueAsString(value)); - System.out.println("NEW VALUE " + getObjectMapper().writeValueAsString(newValue)); - descriptor.getWriteMethod().invoke(newValue); - } - - visit(value, mapping); - } - } - - } catch (final IllegalArgumentException e) { - // handle this please - } catch (final IllegalAccessException e) { - // and this also - } catch (final InvocationTargetException e) { - // and this, too - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - } - } catch (final IntrospectionException e) { - // do something sensible here - } - } - - private static ObjectMapper getObjectMapper() { - final ObjectMapper mapper = new ObjectMapper(); - return mapper; - } - - private static void navigate(Object o, Map> mapping) { - if (Objects.isNull(o) || isPrimitive(o)) { + private static void navigate(Object o, Map> mapping) throws IllegalAccessException { + if (isPrimitive(o)) { return; + } else if (isIterable(o.getClass())) { + for (final Object elem : (Iterable) o) { + navigate(elem, mapping); + } + } else if (hasMapping(o, mapping)) { + mapping.get(o.getClass()).accept(o); } else { - try { - for (Field field : getAllFields(o.getClass())) { - System.out.println(field.getName()); - field.setAccessible(true); - Object value = field.get(o); - - if (Objects.nonNull(value)) { - final Class fieldType = field.getType(); - if ((fieldType.isArray() && !fieldType.getComponentType().isPrimitive())) { - Object[] fs = (Object[]) value; - for (Object fi : fs) { - navigate(fi, mapping); - } - } - if (Iterable.class.isAssignableFrom(fieldType)) { - Iterable fs = (Iterable) value; - for (Object fi : fs) { - navigate(fi, mapping); - } - } else { - if (mapping.keySet().contains(value.getClass())) { - System.out.println("PATCHING " + field.getName()); - field.set(o, mapping.get(value.getClass()).apply(value)); - } - } - } + for (final Field f : getAllFields(o.getClass())) { + f.setAccessible(true); + final Object val = f.get(o); + if (!isPrimitive(val) && hasMapping(val, mapping)) { + mapping.get(val.getClass()).accept(val); + } else { + navigate(f.get(o), mapping); } - - } catch (IllegalAccessException | IllegalArgumentException e) { - throw new RuntimeException(e); } } } + private static boolean hasMapping(Object o, Map> mapping) { + return mapping.containsKey(o.getClass()); + } + + private static boolean isIterable(final Class cl) { + return Iterable.class.isAssignableFrom(cl); + } + private static boolean isPrimitive(Object o) { - return o.getClass().isPrimitive() + return Objects.isNull(o) + || o.getClass().isPrimitive() || o instanceof Class || o instanceof Integer || o instanceof Double diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java deleted file mode 100644 index 264c95daf..000000000 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator2.java +++ /dev/null @@ -1,96 +0,0 @@ - -package eu.dnetlib.dhp.oa.graph.clean; - -import java.lang.reflect.Field; -import java.util.*; -import java.util.function.Function; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import eu.dnetlib.dhp.schema.oaf.Oaf; - -public class OafNavigator2 { - - public static E apply(E oaf, Map> mapping) { - navigate(oaf, mapping); - return oaf; - } - - private static void navigate(Object o, Map> mapping) { - if (Objects.isNull(o) || isPrimitive(o)) { - return; - } else { - try { - for (Field field : getAllFields(o.getClass())) { - //System.out.println("VISITING " + field.getName() + " in " + o.getClass()); - field.setAccessible(true); - Object value = field.get(o); - - if (Objects.nonNull(value)) { - final Class fieldType = field.getType(); - if ((fieldType.isArray() && !fieldType.getComponentType().isPrimitive())) { - Object[] fs = (Object[]) value; - for (Object fi : fs) { - navigate(fi, mapping); - } - } - if (Iterable.class.isAssignableFrom(fieldType)) { - Iterable fs = (Iterable) value; - for (Object fi : fs) { - navigate(fi, mapping); - } - } else { - final Function cleaningFn = mapping.get(value.getClass()); - if (Objects.nonNull(cleaningFn)) { - final Object newValue = cleaningFn.apply(value); - if (!Objects.equals(value, newValue)) { - //System.out.println("PATCHING " + field.getName() + " " + value.getClass()); - //System.out.println("OLD VALUE " + getObjectMapper().writeValueAsString(value)); - //System.out.println("NEW VALUE " + getObjectMapper().writeValueAsString(newValue)); - field.set(o, newValue); - } - } - } - } - } - - } catch (IllegalAccessException | IllegalArgumentException /*| JsonProcessingException*/ e) { - throw new RuntimeException(e); - } - } - } - - private static ObjectMapper getObjectMapper() { - final ObjectMapper mapper = new ObjectMapper(); - return mapper; - } - - private static boolean isPrimitive(Object o) { - return o.getClass().isPrimitive() - || o instanceof Class - || o instanceof Integer - || o instanceof Double - || o instanceof Float - || o instanceof Long - || o instanceof Boolean - || o instanceof String - || o instanceof Byte; - } - - private static List getAllFields(Class clazz) { - return getAllFields(new LinkedList<>(), clazz); - } - - private static List getAllFields(List fields, Class clazz) { - fields.addAll(Arrays.asList(clazz.getDeclaredFields())); - - final Class superclass = clazz.getSuperclass(); - if (Objects.nonNull(superclass) && superclass.getPackage().equals(Oaf.class.getPackage())) { - getAllFields(fields, superclass); - } - - return fields; - } - -} From 0d52816244b09a2ace1f3afb8577abcea4ea2e2e Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sat, 13 Jun 2020 13:06:04 +0200 Subject: [PATCH 78/88] WIP: graph cleaner implementation --- .../common/FunctionalInterfaceSupport.java | 11 +++ ...roperties.java => CleanGraphSparkJob.java} | 24 ++----- ...CleaningRule.java => CleaningRuleMap.java} | 41 +++-------- .../{OafNavigator.java => OafCleaner.java} | 15 ++-- .../clean/{ => oozie_app}/config-default.xml | 0 .../graph/clean/{ => oozie_app}/workflow.xml | 72 +++++++++---------- ...uleTest.java => CleaningFunctionTest.java} | 20 +++--- 7 files changed, 76 insertions(+), 107 deletions(-) rename dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/{CleanGraphProperties.java => CleanGraphSparkJob.java} (84%) rename dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/{CleaningRule.java => CleaningRuleMap.java} (50%) rename dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/{OafNavigator.java => OafCleaner.java} (80%) rename dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/{ => oozie_app}/config-default.xml (100%) rename dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/{ => oozie_app}/workflow.xml (80%) rename dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/{CleaningRuleTest.java => CleaningFunctionTest.java} (82%) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java index e793e3f29..c6c9d8044 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java @@ -2,6 +2,7 @@ package eu.dnetlib.dhp.common; import java.io.Serializable; +import java.util.function.Consumer; import java.util.function.Supplier; /** Provides serializable and throwing extensions to standard functional interfaces. */ @@ -10,6 +11,16 @@ public class FunctionalInterfaceSupport { private FunctionalInterfaceSupport() { } + /** + * Serializable consumer of any kind of objects. To be used withing spark processing pipelines when supplying + * functions externally. + * + * @param + */ + @FunctionalInterface + public interface SerializableConsumer extends Consumer, Serializable { + } + /** * Serializable supplier of any kind of objects. To be used withing spark processing pipelines when supplying * functions externally. diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphProperties.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJob.java similarity index 84% rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphProperties.java rename to dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJob.java index 86dca6d21..b2c7152d5 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphProperties.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJob.java @@ -3,19 +3,9 @@ package eu.dnetlib.dhp.oa.graph.clean; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import java.beans.BeanInfo; -import java.beans.IntrospectionException; -import java.beans.Introspector; -import java.beans.PropertyDescriptor; -import java.lang.reflect.InvocationTargetException; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.TreeMap; -import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; @@ -33,11 +23,10 @@ import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; -import scala.Predef; -public class CleanGraphProperties { +public class CleanGraphSparkJob { - private static final Logger log = LoggerFactory.getLogger(CleanGraphProperties.class); + private static final Logger log = LoggerFactory.getLogger(CleanGraphSparkJob.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -45,7 +34,7 @@ public class CleanGraphProperties { String jsonConfiguration = IOUtils .toString( - CleanGraphProperties.class + CleanGraphSparkJob.class .getResourceAsStream( "/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json")); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); @@ -91,13 +80,14 @@ public class CleanGraphProperties { Class clazz, String outputPath) { - CleaningRule rule = new CleaningRule<>(vocs); + final CleaningRuleMap mapping = CleaningRuleMap.create(vocs); readTableFromPath(spark, inputPath, clazz) - .map(rule, Encoders.bean(clazz)) + .map((MapFunction) value -> OafCleaner.apply(value, mapping), Encoders.bean(clazz)) .write() .mode(SaveMode.Overwrite) - .parquet(outputPath); + .option("compression", "gzip") + .json(outputPath); } private static Dataset readTableFromPath( diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java similarity index 50% rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java rename to dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java index 6d7a262be..8006f7300 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRule.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java @@ -1,43 +1,23 @@ package eu.dnetlib.dhp.oa.graph.clean; -import java.util.Map; -import java.util.function.Consumer; - -import org.apache.spark.api.java.function.MapFunction; - -import com.google.common.collect.Maps; +import java.io.Serializable; +import java.util.HashMap; +import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.SerializableConsumer; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; -import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Qualifier; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; -public class CleaningRule implements MapFunction { - - private VocabularyGroup vocabularies; - - private Map> mapping = Maps.newHashMap(); - - public CleaningRule(VocabularyGroup vocabularies) { - this.vocabularies = vocabularies; - setMappings(vocabularies); - } - - @Override - public T call(T value) throws Exception { - - OafNavigator.apply(value, mapping); - - return value; - } +public class CleaningRuleMap extends HashMap> implements Serializable { /** - * Populates the mapping for the Oaf types subject to cleaning - * + * Creates the mapping for the Oaf types subject to cleaning + * * @param vocabularies */ - private void setMappings(VocabularyGroup vocabularies) { + public static CleaningRuleMap create(VocabularyGroup vocabularies) { + CleaningRuleMap mapping = new CleaningRuleMap(); mapping.put(Qualifier.class, o -> { Qualifier q = (Qualifier) o; if (vocabularies.vocabularyExists(q.getSchemeid())) { @@ -54,10 +34,7 @@ public class CleaningRule implements MapFunction { * } */ }); - } - - public VocabularyGroup getVocabularies() { - return vocabularies; + return mapping; } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java similarity index 80% rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java rename to dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java index 2cc499577..9ba153ba5 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafNavigator.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java @@ -1,15 +1,18 @@ package eu.dnetlib.dhp.oa.graph.clean; +import java.io.Serializable; import java.lang.reflect.Field; -import java.util.*; -import java.util.function.Consumer; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; import eu.dnetlib.dhp.schema.oaf.Oaf; -public class OafNavigator { +public class OafCleaner implements Serializable { - public static E apply(E oaf, Map> mapping) { + public static E apply(E oaf, CleaningRuleMap mapping) { try { navigate(oaf, mapping); } catch (IllegalAccessException e) { @@ -18,7 +21,7 @@ public class OafNavigator { return oaf; } - private static void navigate(Object o, Map> mapping) throws IllegalAccessException { + private static void navigate(Object o, CleaningRuleMap mapping) throws IllegalAccessException { if (isPrimitive(o)) { return; } else if (isIterable(o.getClass())) { @@ -40,7 +43,7 @@ public class OafNavigator { } } - private static boolean hasMapping(Object o, Map> mapping) { + private static boolean hasMapping(Object o, CleaningRuleMap mapping) { return mapping.containsKey(o.getClass()); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/config-default.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/config-default.xml similarity index 100% rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/config-default.xml rename to dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/config-default.xml diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml similarity index 80% rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/workflow.xml rename to dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml index d152448af..7329df29a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml @@ -72,18 +72,17 @@ yarn cluster Clean publications - eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob dhp-graph-mapper-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} - --driver-memory=${sparkDriverMemoryForJoining} + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=7680 - --conf spark.network.timeout=${sparkNetworkTimeout} --inputPath${graphInputPath}/publication --outputPath${graphOutputPath}/publication @@ -99,18 +98,17 @@ yarn cluster Clean datasets - eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob dhp-graph-mapper-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} - --driver-memory=${sparkDriverMemoryForJoining} + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=7680 - --conf spark.network.timeout=${sparkNetworkTimeout} --inputPath${graphInputPath}/dataset --outputPath${graphOutputPath}/dataset @@ -126,18 +124,17 @@ yarn cluster Clean otherresearchproducts - eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob dhp-graph-mapper-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} - --driver-memory=${sparkDriverMemoryForJoining} + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=7680 - --conf spark.network.timeout=${sparkNetworkTimeout} --inputPath${graphInputPath}/otherresearchproduct --outputPath${graphOutputPath}/otherresearchproduct @@ -153,18 +150,17 @@ yarn cluster Clean softwares - eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob dhp-graph-mapper-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} - --driver-memory=${sparkDriverMemoryForJoining} + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=7680 - --conf spark.network.timeout=${sparkNetworkTimeout} --inputPath${graphInputPath}/software --outputPath${graphOutputPath}/software @@ -180,18 +176,17 @@ yarn cluster Clean datasources - eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob dhp-graph-mapper-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} - --driver-memory=${sparkDriverMemoryForJoining} + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=7680 - --conf spark.network.timeout=${sparkNetworkTimeout} --inputPath${graphInputPath}/datasource --outputPath${graphOutputPath}/datasource @@ -207,18 +202,17 @@ yarn cluster Clean organizations - eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob dhp-graph-mapper-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} - --driver-memory=${sparkDriverMemoryForJoining} + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=7680 - --conf spark.network.timeout=${sparkNetworkTimeout} --inputPath${graphInputPath}/organization --outputPath${graphOutputPath}/organization @@ -234,18 +228,17 @@ yarn cluster Clean projects - eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob dhp-graph-mapper-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} - --driver-memory=${sparkDriverMemoryForJoining} + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=7680 - --conf spark.network.timeout=${sparkNetworkTimeout} --inputPath${graphInputPath}/project --outputPath${graphOutputPath}/project @@ -261,18 +254,17 @@ yarn cluster Clean relations - eu.dnetlib.dhp.oa.graph.clean.CleanGraphProperties + eu.dnetlib.dhp.oa.graph.clean.CleanGraphSparkJob dhp-graph-mapper-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} - --driver-memory=${sparkDriverMemoryForJoining} + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=7680 - --conf spark.network.timeout=${sparkNetworkTimeout} --inputPath${graphInputPath}/relation --outputPath${graphOutputPath}/relation diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java similarity index 82% rename from dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java rename to dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java index 7792f64c6..d1f152342 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java @@ -5,32 +5,27 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.lenient; import java.io.IOException; -import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.platform.commons.util.StringUtils; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; -import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyTerm; import eu.dnetlib.dhp.schema.oaf.Publication; import eu.dnetlib.dhp.schema.oaf.Qualifier; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; @ExtendWith(MockitoExtension.class) -public class CleaningRuleTest { +public class CleaningFunctionTest { public static final ObjectMapper MAPPER = new ObjectMapper(); @@ -39,7 +34,7 @@ public class CleaningRuleTest { private VocabularyGroup vocabularies; - private CleaningRule cleaningRule; + private CleaningRuleMap mapping; @BeforeEach public void setUp() throws ISLookUpException, IOException { @@ -49,18 +44,19 @@ public class CleaningRuleTest { .thenReturn(synonyms()); vocabularies = VocabularyGroup.loadVocsFromIS(isLookUpService); - cleaningRule = new CleaningRule(vocabularies); + mapping = CleaningRuleMap.create(vocabularies); } @Test public void testCleaning() throws Exception { - assertNotNull(cleaningRule.getVocabularies()); + assertNotNull(vocabularies); + assertNotNull(mapping); String json = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/result.json")); Publication p_in = MAPPER.readValue(json, Publication.class); - Publication p_out = cleaningRule.call(p_in); + Publication p_out = OafCleaner.apply(p_in, mapping); assertNotNull(p_out); @@ -100,11 +96,11 @@ public class CleaningRuleTest { private List vocs() throws IOException { return IOUtils - .readLines(CleaningRuleTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/terms.txt")); + .readLines(CleaningFunctionTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/terms.txt")); } private List synonyms() throws IOException { return IOUtils - .readLines(CleaningRuleTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt")); + .readLines(CleaningFunctionTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt")); } } From c15c8c0ad0ae0b6c3b86616051c10531ccfa6946 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 15 Jun 2020 16:07:30 +0200 Subject: [PATCH 79/88] map datasource identities (including piwik ids) as original IDs --- .../dhp/oa/graph/raw/MigrateDbEntitiesApplication.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java index 89bf5c379..0bad89e9e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java @@ -50,6 +50,8 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Lists; + import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.DbClient; import eu.dnetlib.dhp.oa.graph.raw.common.AbstractMigrationApplication; @@ -171,7 +173,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i final Datasource ds = new Datasource(); ds.setId(createOpenaireId(10, rs.getString("datasourceid"), true)); - ds.setOriginalId(Arrays.asList(rs.getString("datasourceid"))); + ds.setOriginalId(Arrays.asList((String[]) rs.getArray("identities").getArray())); ds .setCollectedfrom( listKeyValues( From 2a4f65795f981faeac358875f89ce450e905b88f Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 15 Jun 2020 18:32:24 +0200 Subject: [PATCH 80/88] WIP: graph cleaner implementation --- .../dhp/oa/graph/raw/common/Vocabulary.java | 2 +- .../oa/graph/raw/common/VocabularyGroup.java | 9 +- .../oa/graph/clean/CleaningFunctionTest.java | 4 +- .../eu/dnetlib/dhp/oa/graph/clean/result.json | 8 +- .../dnetlib/dhp/oa/graph/clean/synonyms.txt | 1592 ++++++++--------- .../eu/dnetlib/dhp/oa/graph/clean/terms.txt | 1197 +++++++------ 6 files changed, 1400 insertions(+), 1412 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java index ffa64861c..9bf198c8b 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java @@ -61,7 +61,7 @@ public class Vocabulary implements Serializable { } public VocabularyTerm getTermBySynonym(final String syn) { - return getTerm(synonyms.get(syn)); + return getTerm(synonyms.get(syn.toLowerCase())); } public Qualifier getTermAsQualifier(final String termId) { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java index 447228bf9..d9ff62596 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java @@ -3,7 +3,6 @@ package eu.dnetlib.dhp.oa.graph.raw.common; import java.io.Serializable; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -14,7 +13,7 @@ import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; public class VocabularyGroup implements Serializable { - public static final String VOCABULARIES_XQUERY = "for $x in collection(' /db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') \n" + public static final String VOCABULARIES_XQUERY = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') \n" + "let $vocid := $x//VOCABULARY_NAME/@code\n" + "let $vocname := $x//VOCABULARY_NAME/text()\n" + @@ -46,7 +45,7 @@ public class VocabularyGroup implements Serializable { } vocs.addTerm(vocId, termId, termName); - vocs.addSynonyms(vocId, termId, termId); + // vocs.addSynonyms(vocId, termId, termId); } } @@ -58,7 +57,7 @@ public class VocabularyGroup implements Serializable { final String syn = arr[2].trim(); vocs.addSynonyms(vocId, termId, syn); - vocs.addSynonyms(vocId, termId, termId); + // vocs.addSynonyms(vocId, termId, termId); } } @@ -135,7 +134,7 @@ public class VocabularyGroup implements Serializable { Optional .ofNullable(vocs.get(id)) .orElseThrow(() -> new IllegalArgumentException("missing vocabulary id: " + vocId)) - .addSynonym(syn, termId); + .addSynonym(syn.toLowerCase(), termId); } } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java index d1f152342..1b21ce2d3 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java @@ -60,8 +60,8 @@ public class CleaningFunctionTest { assertNotNull(p_out); - assertEquals("eng", p_out.getLanguage().getClassid()); - assertEquals("English", p_out.getLanguage().getClassname()); + assertEquals("und", p_out.getLanguage().getClassid()); + assertEquals("Undetermined", p_out.getLanguage().getClassname()); assertEquals("0018", p_out.getInstance().get(0).getInstancetype().getClassid()); assertEquals("Annotation", p_out.getInstance().get(0).getInstancetype().getClassname()); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json index 89ebe5af5..b63a12f61 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json @@ -281,8 +281,8 @@ "value": "VIRTA" }, "instancetype": { - "classid": "Comentario", - "classname": "Comentario", + "classid": "Comment/debate", + "classname": "Comment/debate", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource" }, @@ -317,8 +317,8 @@ "vol": "" }, "language": { - "classid": "en", - "classname": "en", + "classid": "UNKNOWN", + "classname": "UNKNOWN", "schemeid": "dnet:languages", "schemename": "dnet:languages" }, diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt index 10eba3e86..05484c8e5 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt @@ -1,223 +1,9 @@ -dnet:countries @=@ AF @=@ AFG -dnet:countries @=@ AF @=@ Afghanistan -dnet:countries @=@ AD @=@ Andorra -dnet:countries @=@ AO @=@ Angola -dnet:countries @=@ AR @=@ ARG -dnet:countries @=@ AR @=@ Argentina -dnet:countries @=@ AU @=@ AUS -dnet:countries @=@ AU @=@ Australia -dnet:countries @=@ AT @=@ AUT -dnet:countries @=@ AT @=@ Austria -dnet:countries @=@ AZ @=@ AZE -dnet:countries @=@ BD @=@ Bangladesh -dnet:countries @=@ BY @=@ Belarus -dnet:countries @=@ BE @=@ BEL -dnet:countries @=@ BE @=@ Belgium -dnet:countries @=@ BJ @=@ BEN -dnet:countries @=@ BO @=@ Bolivia, Plurinational State of -dnet:countries @=@ BA @=@ BIH -dnet:countries @=@ BA @=@ Bosnia-Hercegovina -dnet:countries @=@ BR @=@ BRA -dnet:countries @=@ BR @=@ Brazil -dnet:countries @=@ BG @=@ Bulgaria -dnet:countries @=@ BF @=@ BFA -dnet:countries @=@ KH @=@ Campuchea -dnet:countries @=@ CM @=@ CMR -dnet:countries @=@ CA @=@ CAN -dnet:countries @=@ CA @=@ Canada -dnet:countries @=@ CV @=@ Cape Verde -dnet:countries @=@ CL @=@ CHL -dnet:countries @=@ CL @=@ Chile -dnet:countries @=@ CN @=@ CHN -dnet:countries @=@ CN @=@ China -dnet:countries @=@ CO @=@ COL -dnet:countries @=@ CO @=@ Colombia -dnet:countries @=@ CG @=@ Congo, Republic -dnet:countries @=@ CD @=@ Congo, the Democratic Republic of the -dnet:countries @=@ CD @=@ Congo Democratic Republic (formerly Zaire) -dnet:countries @=@ CD @=@ Congo Democratic Republic -dnet:countries @=@ CD @=@ Zaire -dnet:countries @=@ CR @=@ CRI -dnet:countries @=@ CI @=@ CIV -dnet:countries @=@ CI @=@ Ivory Coast -dnet:countries @=@ HR @=@ Croatia -dnet:countries @=@ HR @=@ HRV -dnet:countries @=@ CY @=@ CYP -dnet:countries @=@ CY @=@ Cyprus -dnet:countries @=@ CZ @=@ CZE -dnet:countries @=@ CZ @=@ Czech Republic -dnet:countries @=@ CZ @=@ Czechoslovakia -dnet:countries @=@ DK @=@ DNK -dnet:countries @=@ DK @=@ Denmark -dnet:countries @=@ EC @=@ Ecuador -dnet:countries @=@ EG @=@ EGY -dnet:countries @=@ EG @=@ Egypt -dnet:countries @=@ SV @=@ SLV -dnet:countries @=@ EE @=@ EST -dnet:countries @=@ EE @=@ Estonia -dnet:countries @=@ ET @=@ ETH -dnet:countries @=@ EU @=@ EEC -dnet:countries @=@ FI @=@ FIN -dnet:countries @=@ FI @=@ Finland -dnet:countries @=@ MK @=@ Macedonia, the Former Yugoslav Republic Of -dnet:countries @=@ MK @=@ Macedonia -dnet:countries @=@ FR @=@ FRA -dnet:countries @=@ FR @=@ France -dnet:countries @=@ PF @=@ French Polynesia -dnet:countries @=@ PF @=@ PYF -dnet:countries @=@ TF @=@ French Southern Territories -dnet:countries @=@ GE @=@ Georgia -dnet:countries @=@ DE @=@ DEU -dnet:countries @=@ DE @=@ Germany -dnet:countries @=@ DE @=@ Germany, Berlin -dnet:countries @=@ GH @=@ GHA -dnet:countries @=@ GR @=@ EL -dnet:countries @=@ GR @=@ GRC -dnet:countries @=@ GL @=@ GRL -dnet:countries @=@ GN @=@ Guinea -dnet:countries @=@ GW @=@ Guinea-Bissau -dnet:countries @=@ VA @=@ Vatican State -dnet:countries @=@ HK @=@ HKG -dnet:countries @=@ HK @=@ Hong Kong -dnet:countries @=@ HK @=@ Hongkong -dnet:countries @=@ HU @=@ HUN -dnet:countries @=@ HU @=@ Hungary -dnet:countries @=@ IS @=@ ISL -dnet:countries @=@ IN @=@ IND -dnet:countries @=@ IN @=@ India -dnet:countries @=@ ID @=@ IDN -dnet:countries @=@ ID @=@ Indonesia -dnet:countries @=@ IR @=@ Iran -dnet:countries @=@ IR @=@ Iran, Islamic Republic of -dnet:countries @=@ IE @=@ IRL -dnet:countries @=@ IE @=@ Ireland -dnet:countries @=@ IL @=@ ISR -dnet:countries @=@ IL @=@ Israel -dnet:countries @=@ IT @=@ ITA -dnet:countries @=@ IT @=@ Italy -dnet:countries @=@ JM @=@ Jamaica -dnet:countries @=@ JP @=@ JPN -dnet:countries @=@ JP @=@ Japan -dnet:countries @=@ KZ @=@ KAZ -dnet:countries @=@ KZ @=@ Kazakistan -dnet:countries @=@ KZ @=@ Kazakstan -dnet:countries @=@ KE @=@ KEN -dnet:countries @=@ KE @=@ Kenya -dnet:countries @=@ KR @=@ KOR -dnet:countries @=@ KR @=@ Korea, Republic of -dnet:countries @=@ KR @=@ Korean Republic (South Korea) -dnet:countries @=@ KP @=@ PRK -dnet:countries @=@ LY @=@ Libya -dnet:countries @=@ LT @=@ LTU -dnet:countries @=@ LU @=@ LUX -dnet:countries @=@ LU @=@ Luxembourg -dnet:countries @=@ MO @=@ Macao -dnet:countries @=@ MG @=@ Madagascar -dnet:countries @=@ MY @=@ Malaysia -dnet:countries @=@ ML @=@ Mali -dnet:countries @=@ MT @=@ Malta -dnet:countries @=@ MU @=@ Mauritius -dnet:countries @=@ MX @=@ MEX -dnet:countries @=@ MX @=@ Mexico -dnet:countries @=@ FM @=@ Micronesia -dnet:countries @=@ MD @=@ Moldova -dnet:countries @=@ MD @=@ Moldova, Republic of -dnet:countries @=@ MN @=@ Mongolia -dnet:countries @=@ MA @=@ Morocco -dnet:countries @=@ MZ @=@ Mozambique -dnet:countries @=@ NA @=@ NAM -dnet:countries @=@ NL @=@ NLD -dnet:countries @=@ NL @=@ Netherlands -dnet:countries @=@ AN @=@ Netherlands Antilles -dnet:countries @=@ NZ @=@ NZL -dnet:countries @=@ NZ @=@ New Zealand -dnet:countries @=@ NO @=@ NOR -dnet:countries @=@ NO @=@ Norway -dnet:countries @=@ OC @=@ Australasia -dnet:countries @=@ OM @=@ Oman -dnet:countries @=@ PK @=@ PAK -dnet:countries @=@ PK @=@ Pakistan -dnet:countries @=@ PS @=@ Palestine, State of -dnet:countries @=@ PS @=@ Palestinian Territory, Occupied -dnet:countries @=@ PA @=@ PAN -dnet:countries @=@ PA @=@ Panama -dnet:countries @=@ PG @=@ PapuaNew Guinea -dnet:countries @=@ PE @=@ PER -dnet:countries @=@ PH @=@ PHL -dnet:countries @=@ PH @=@ Philippines -dnet:countries @=@ PL @=@ POL -dnet:countries @=@ PL @=@ Poland -dnet:countries @=@ PT @=@ PRT -dnet:countries @=@ PT @=@ Portugal -dnet:countries @=@ PR @=@ Puerto Rico -dnet:countries @=@ RO @=@ ROU -dnet:countries @=@ RO @=@ Romania -dnet:countries @=@ RU @=@ RUS -dnet:countries @=@ RU @=@ Russia -dnet:countries @=@ RU @=@ Russian Federation -dnet:countries @=@ RE @=@ Réunion -dnet:countries @=@ KN @=@ Saint Kitts And Nevis -dnet:countries @=@ SA @=@ Saudi Arabia -dnet:countries @=@ SN @=@ SEN -dnet:countries @=@ SG @=@ SGP -dnet:countries @=@ SG @=@ Singapore -dnet:countries @=@ SK @=@ SVK -dnet:countries @=@ SI @=@ SVN -dnet:countries @=@ SI @=@ Slovenia -dnet:countries @=@ ZA @=@ South Africa -dnet:countries @=@ ZA @=@ ZAF -dnet:countries @=@ ES @=@ ESP -dnet:countries @=@ ES @=@ Spain -dnet:countries @=@ LK @=@ Sri Lanka -dnet:countries @=@ LK @=@ LKA -dnet:countries @=@ SR @=@ Suriname -dnet:countries @=@ SE @=@ SWE -dnet:countries @=@ SE @=@ Sweden -dnet:countries @=@ CH @=@ CHE -dnet:countries @=@ CH @=@ Switzerland -dnet:countries @=@ SY @=@ Syria -dnet:countries @=@ ST @=@ Sao Tome and Principe -dnet:countries @=@ TW @=@ TWN -dnet:countries @=@ TW @=@ Taiwan, Province of China -dnet:countries @=@ TZ @=@ Tanzania -dnet:countries @=@ TZ @=@ Tanzania, United Republic of -dnet:countries @=@ TH @=@ THA -dnet:countries @=@ TH @=@ Thailand -dnet:countries @=@ TL @=@ East Timor -dnet:countries @=@ TN @=@ TUN -dnet:countries @=@ TN @=@ Tunisia -dnet:countries @=@ TR @=@ TUR -dnet:countries @=@ TR @=@ Turkey -dnet:countries @=@ UNKNOWN @=@ AAA -dnet:countries @=@ UNKNOWN @=@ [Unknown] -dnet:countries @=@ UNKNOWN @=@ _? -dnet:countries @=@ UA @=@ UKR -dnet:countries @=@ UA @=@ Ukraine -dnet:countries @=@ AE @=@ United Arab Emirates -dnet:countries @=@ GB @=@ GBR -dnet:countries @=@ GB @=@ UK -dnet:countries @=@ GB @=@ United Kingdom -dnet:countries @=@ GB @=@ Great Britain -dnet:countries @=@ GB @=@ Great Britain and Northern Ireland -dnet:countries @=@ US @=@ USA -dnet:countries @=@ US @=@ United States -dnet:countries @=@ US @=@ United States of America -dnet:countries @=@ UY @=@ Uruguay -dnet:countries @=@ UZ @=@ Uzbekistan -dnet:countries @=@ VE @=@ Venezuela, Bolivarian Republic of -dnet:countries @=@ VN @=@ Vietnam -dnet:countries @=@ VG @=@ British Virgin Islands -dnet:countries @=@ YU @=@ Jugoslavia -dnet:countries @=@ YU @=@ Yugoslavia -dnet:countries @=@ ZW @=@ ABW -dnet:datasourceCompatibilityLevel @=@ openaire-pub_4.0 @=@ openaire4.0 dnet:access_modes @=@ CLOSED @=@ http://purl.org/coar/access_right/c_14cb dnet:access_modes @=@ CLOSED @=@ info:eu-repo/semantics/closedAccess dnet:access_modes @=@ EMBARGO @=@ http://purl.org/coar/access_right/c_f1cf dnet:access_modes @=@ EMBARGO @=@ info:eu-repo/semantics/embargoedAccess dnet:access_modes @=@ OPEN @=@ Creative Commons License [CC BY-NC-ND] http://creativecommons.org/licenses/by-nc-nd/3.0/de/ dnet:access_modes @=@ OPEN @=@ Creative commons -dnet:access_modes @=@ OPEN @=@ Open Access dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-nc-nd/3.0/ dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-nc/3.0/ dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-sa/3.0/ @@ -228,53 +14,489 @@ dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by/4.0/ dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/publicdomain/zero/1.0/ dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/publicdomain/zero/1.0/ & http://www.canadensys.net/norms dnet:access_modes @=@ OPEN @=@ http://purl.org/coar/access_right/c_abf2 +dnet:access_modes @=@ OPEN @=@ https://creativecommons.org/licenses/by-nc/4.0/ dnet:access_modes @=@ OPEN @=@ info:eu-repo/semantics/openAccess -dnet:access_modes @=@ OPEN @=@ openAccess -dnet:access_modes @=@ OPEN @=@ http://creativecommons.org/licenses/by-nc-nd/2.0/uk/ +dnet:access_modes @=@ OPEN @=@ open_access dnet:access_modes @=@ RESTRICTED @=@ http://purl.org/coar/access_right/c_16ec dnet:access_modes @=@ RESTRICTED @=@ info:eu-repo/semantics/restrictedAccess -dnet:result_typologies @=@ dataset @=@ 0021 -dnet:result_typologies @=@ dataset @=@ 0024 -dnet:result_typologies @=@ dataset @=@ 0025 -dnet:result_typologies @=@ dataset @=@ 0030 -dnet:result_typologies @=@ dataset @=@ 0033 -dnet:result_typologies @=@ dataset @=@ 0037 -dnet:result_typologies @=@ dataset @=@ 0039 -dnet:result_typologies @=@ other @=@ 0000 -dnet:result_typologies @=@ other @=@ 0010 -dnet:result_typologies @=@ other @=@ 0018 -dnet:result_typologies @=@ other @=@ 0020 -dnet:result_typologies @=@ other @=@ 0022 -dnet:result_typologies @=@ other @=@ 0023 -dnet:result_typologies @=@ other @=@ 0026 -dnet:result_typologies @=@ other @=@ 0027 -dnet:result_typologies @=@ other @=@ 0028 -dnet:result_typologies @=@ publication @=@ 0001 -dnet:result_typologies @=@ publication @=@ 0002 -dnet:result_typologies @=@ publication @=@ 0004 -dnet:result_typologies @=@ publication @=@ 0005 -dnet:result_typologies @=@ publication @=@ 0006 -dnet:result_typologies @=@ publication @=@ 0007 -dnet:result_typologies @=@ publication @=@ 0008 -dnet:result_typologies @=@ publication @=@ 0009 -dnet:result_typologies @=@ publication @=@ 0011 -dnet:result_typologies @=@ publication @=@ 0012 -dnet:result_typologies @=@ publication @=@ 0013 -dnet:result_typologies @=@ publication @=@ 0014 -dnet:result_typologies @=@ publication @=@ 0015 -dnet:result_typologies @=@ publication @=@ 0016 -dnet:result_typologies @=@ publication @=@ 0017 -dnet:result_typologies @=@ publication @=@ 0019 -dnet:result_typologies @=@ publication @=@ 0031 -dnet:result_typologies @=@ publication @=@ 0032 -dnet:result_typologies @=@ publication @=@ 0034 -dnet:result_typologies @=@ publication @=@ 0035 -dnet:result_typologies @=@ publication @=@ 0036 -dnet:result_typologies @=@ publication @=@ 0038 -dnet:result_typologies @=@ publication @=@ 0044 -dnet:result_typologies @=@ publication @=@ 0045 -dnet:result_typologies @=@ software @=@ 0029 -dnet:result_typologies @=@ software @=@ 0040 +dnet:compatibilityLevel @=@ openaire-pub_4.0 @=@ openaire4.0 +dnet:subject_classification_typologies @=@ jel @=@ jelElement +dnet:publication_resource @=@ 0018 @=@ Comment/debate +dnet:publication_resource @=@ 0018 @=@ http://purl.org/coar/resource_type/c_1162 +dnet:publication_resource @=@ 0018 @=@ info:eu-repo/semantics/annotation +dnet:publication_resource @=@ 0001 @=@ A1 Alkuperäisartikkeli tieteellisessä aikakauslehdessä +dnet:publication_resource @=@ 0001 @=@ Article +dnet:publication_resource @=@ 0001 @=@ Article (author) +dnet:publication_resource @=@ 0001 @=@ Article - letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article / Letter to editor +dnet:publication_resource @=@ 0001 @=@ Article / Letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article / Newspaper +dnet:publication_resource @=@ 0001 @=@ Article in journal +dnet:publication_resource @=@ 0001 @=@ Article in monograph or in proceedings +dnet:publication_resource @=@ 0001 @=@ Article in proceedings +dnet:publication_resource @=@ 0001 @=@ Article-letter to the editor +dnet:publication_resource @=@ 0001 @=@ Article/Letter to editor +dnet:publication_resource @=@ 0001 @=@ Articolo +dnet:publication_resource @=@ 0001 @=@ Artículo +dnet:publication_resource @=@ 0001 @=@ Aufsatz +dnet:publication_resource @=@ 0001 @=@ Clinical Study +dnet:publication_resource @=@ 0001 @=@ Institutional Series +dnet:publication_resource @=@ 0001 @=@ International Journal +dnet:publication_resource @=@ 0001 @=@ International Journal Abstract +dnet:publication_resource @=@ 0001 @=@ International Journal ISI/JCR +dnet:publication_resource @=@ 0001 @=@ Journal (full / special issue) +dnet:publication_resource @=@ 0001 @=@ Journal Article/Review +dnet:publication_resource @=@ 0001 @=@ Journal article +dnet:publication_resource @=@ 0001 @=@ Journal article (on-line or printed) +dnet:publication_resource @=@ 0001 @=@ Journal articles +dnet:publication_resource @=@ 0001 @=@ Journal paper +dnet:publication_resource @=@ 0001 @=@ National Journal +dnet:publication_resource @=@ 0001 @=@ Original article (non peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Original article (peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Peer-reviewed Article +dnet:publication_resource @=@ 0001 @=@ Published Journal Article +dnet:publication_resource @=@ 0001 @=@ Research Article +dnet:publication_resource @=@ 0001 @=@ Review article (non peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Review article (peer-reviewed) +dnet:publication_resource @=@ 0001 @=@ Volumes Edited / Special Issues +dnet:publication_resource @=@ 0001 @=@ article in non peer-reviewed journal +dnet:publication_resource @=@ 0001 @=@ article in peer-reviewed journal +dnet:publication_resource @=@ 0001 @=@ article-commentary +dnet:publication_resource @=@ 0001 @=@ article_site_web +dnet:publication_resource @=@ 0001 @=@ doc-type:Journal Article +dnet:publication_resource @=@ 0001 @=@ doc-type:article +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_545b +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_6501 +dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_7877 +dnet:publication_resource @=@ 0001 @=@ in-brief +dnet:publication_resource @=@ 0001 @=@ info:eu-repo/semantics/article +dnet:publication_resource @=@ 0001 @=@ journal-article +dnet:publication_resource @=@ 0001 @=@ journalArticle +dnet:publication_resource @=@ 0001 @=@ journal_article +dnet:publication_resource @=@ 0001 @=@ letter +dnet:publication_resource @=@ 0001 @=@ non peer-reviewed article +dnet:publication_resource @=@ 0001 @=@ partial-retraction +dnet:publication_resource @=@ 0001 @=@ proceeding with peer review +dnet:publication_resource @=@ 0001 @=@ publication-article +dnet:publication_resource @=@ 0001 @=@ rapid-communication +dnet:publication_resource @=@ 0001 @=@ reply +dnet:publication_resource @=@ 0001 @=@ research-article +dnet:publication_resource @=@ 0001 @=@ retraction +dnet:publication_resource @=@ 0001 @=@ review-article +dnet:publication_resource @=@ 0001 @=@ text (article) +dnet:publication_resource @=@ 0001 @=@ Статья +dnet:publication_resource @=@ 0001 @=@ ArticleArtikel +dnet:publication_resource @=@ 0033 @=@ AUDIOVISUAL_DOCUMENT +dnet:publication_resource @=@ 0033 @=@ Audiovisual/Audiovisual +dnet:publication_resource @=@ 0033 @=@ http://purl.org/coar/resource_type/c_c513 +dnet:publication_resource @=@ 0008 @=@ Bachelor's +dnet:publication_resource @=@ 0008 @=@ Bachelor's Degree +dnet:publication_resource @=@ 0008 @=@ Bachelors Thesis +dnet:publication_resource @=@ 0008 @=@ Proyecto fin de carrera +dnet:publication_resource @=@ 0008 @=@ Undergraduate Thesis +dnet:publication_resource @=@ 0008 @=@ http://purl.org/coar/resource_type/c_7a1f +dnet:publication_resource @=@ 0008 @=@ info:eu-repo/semantics/bachelorThesis +dnet:publication_resource @=@ 0008 @=@ выпускная бакалаврская работа +dnet:publication_resource @=@ 0002 @=@ Book (monograph) +dnet:publication_resource @=@ 0002 @=@ Book (non peer-reviewed) +dnet:publication_resource @=@ 0002 @=@ Book (peer-reviewed) +dnet:publication_resource @=@ 0002 @=@ Book - monograph - editorial book +dnet:publication_resource @=@ 0002 @=@ Book Section +dnet:publication_resource @=@ 0002 @=@ Book as author +dnet:publication_resource @=@ 0002 @=@ Buch +dnet:publication_resource @=@ 0002 @=@ International Book/Monograph +dnet:publication_resource @=@ 0002 @=@ Libro +dnet:publication_resource @=@ 0002 @=@ Monografia +dnet:publication_resource @=@ 0002 @=@ Monograph +dnet:publication_resource @=@ 0002 @=@ National Book/Monograph +dnet:publication_resource @=@ 0002 @=@ atlas +dnet:publication_resource @=@ 0002 @=@ book +dnet:publication_resource @=@ 0002 @=@ book-series +dnet:publication_resource @=@ 0002 @=@ book-set +dnet:publication_resource @=@ 0002 @=@ book-track +dnet:publication_resource @=@ 0002 @=@ book_series +dnet:publication_resource @=@ 0002 @=@ book_title +dnet:publication_resource @=@ 0002 @=@ doc-type:book +dnet:publication_resource @=@ 0002 @=@ edited-book +dnet:publication_resource @=@ 0002 @=@ http://purl.org/coar/resource_type/c_2f33 +dnet:publication_resource @=@ 0002 @=@ info:eu-repo/semantics/book +dnet:publication_resource @=@ 0002 @=@ ouvrage +dnet:publication_resource @=@ 0002 @=@ publication-book +dnet:publication_resource @=@ 0002 @=@ reference-book +dnet:publication_resource @=@ 0002 @=@ scientific book +dnet:publication_resource @=@ 0002 @=@ Монография +dnet:publication_resource @=@ 0002 @=@ Учебник +dnet:publication_resource @=@ 0037 @=@ clinicalTrial +dnet:publication_resource @=@ 0037 @=@ http://purl.org/coar/resource_type/c_cb28 +dnet:publication_resource @=@ 0022 @=@ collection +dnet:publication_resource @=@ 0004 @=@ A4 Artikkeli konferenssijulkaisussa +dnet:publication_resource @=@ 0004 @=@ Comunicación de congreso +dnet:publication_resource @=@ 0004 @=@ Conference Paper +dnet:publication_resource @=@ 0004 @=@ Conference Paper/Proceeding/Abstract +dnet:publication_resource @=@ 0004 @=@ Conference Proceedings +dnet:publication_resource @=@ 0004 @=@ Conference article +dnet:publication_resource @=@ 0004 @=@ Conference contribution +dnet:publication_resource @=@ 0004 @=@ Conference lecture +dnet:publication_resource @=@ 0004 @=@ Conference or Workshop Item +dnet:publication_resource @=@ 0004 @=@ Conference paper, poster, etc. +dnet:publication_resource @=@ 0004 @=@ Conference papers +dnet:publication_resource @=@ 0004 @=@ Conference report +dnet:publication_resource @=@ 0004 @=@ International Conference +dnet:publication_resource @=@ 0004 @=@ International Conference Abstract/Poster +dnet:publication_resource @=@ 0004 @=@ International Conference ISI/JCR +dnet:publication_resource @=@ 0004 @=@ International Conference communication/abstract/poster +dnet:publication_resource @=@ 0004 @=@ National Conference +dnet:publication_resource @=@ 0004 @=@ National Conference Abstract/Poster +dnet:publication_resource @=@ 0004 @=@ National Conference communication/abstract/poster +dnet:publication_resource @=@ 0004 @=@ PREFACE_PROCEEDINGS +dnet:publication_resource @=@ 0004 @=@ PROCEEDING_PAPER +dnet:publication_resource @=@ 0004 @=@ Papers in Conference Proceedings +dnet:publication_resource @=@ 0004 @=@ Presentación +dnet:publication_resource @=@ 0004 @=@ Proceedings (peer-reviewed) +dnet:publication_resource @=@ 0004 @=@ Proceedings of a Conference +dnet:publication_resource @=@ 0004 @=@ Proceedings paper +dnet:publication_resource @=@ 0004 @=@ Póster +dnet:publication_resource @=@ 0004 @=@ actes_congres +dnet:publication_resource @=@ 0004 @=@ communication_avec_actes +dnet:publication_resource @=@ 0004 @=@ communication_invitee +dnet:publication_resource @=@ 0004 @=@ communication_par_affiche +dnet:publication_resource @=@ 0004 @=@ communication_sans_actes +dnet:publication_resource @=@ 0004 @=@ conference +dnet:publication_resource @=@ 0004 @=@ conference item +dnet:publication_resource @=@ 0004 @=@ conference proceeding +dnet:publication_resource @=@ 0004 @=@ conferenceObject +dnet:publication_resource @=@ 0004 @=@ conference_paper +dnet:publication_resource @=@ 0004 @=@ doc-type:conferenceObject +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_18co +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_18cp +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_5794 +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_6670 +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_c94f +dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_f744 +dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferenceItem +dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferenceObject +dnet:publication_resource @=@ 0004 @=@ invited conference talk +dnet:publication_resource @=@ 0004 @=@ poster +dnet:publication_resource @=@ 0004 @=@ presentation +dnet:publication_resource @=@ 0004 @=@ proceeding, seminar, workshop without peer review +dnet:publication_resource @=@ 0004 @=@ proceedings +dnet:publication_resource @=@ 0004 @=@ proceedings-article +dnet:publication_resource @=@ 0004 @=@ publication-conferencepaper +dnet:publication_resource @=@ 0004 @=@ научный доклад +dnet:publication_resource @=@ 0005 @=@ Newspaper or magazine article +dnet:publication_resource @=@ 0005 @=@ http://purl.org/coar/resource_type/c_998f +dnet:publication_resource @=@ 0005 @=@ info:eu-repo/semantics/contributionToPeriodical +dnet:publication_resource @=@ 0045 @=@ Data Management Plan +dnet:publication_resource @=@ 0045 @=@ Data Management Plan (NSF Generic) +dnet:publication_resource @=@ 0045 @=@ http://purl.org/coar/resource_type/c_ab20 +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataManagementPolicy +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataManagementPolicyDocument +dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataMangementPlan +dnet:publication_resource @=@ 0045 @=@ plan de gestión de datos +dnet:publication_resource @=@ 0045 @=@ publication-datamanagementplan +dnet:publication_resource @=@ 0031 @=@ Data Descriptor +dnet:publication_resource @=@ 0031 @=@ DataPaper +dnet:publication_resource @=@ 0031 @=@ data-article +dnet:publication_resource @=@ 0031 @=@ http://purl.org/coar/resource_type/c_beb9 +dnet:publication_resource @=@ 0021 @=@ Dataset/Dataset +dnet:publication_resource @=@ 0021 @=@ Research Data +dnet:publication_resource @=@ 0021 @=@ dataset +dnet:publication_resource @=@ 0021 @=@ http://purl.org/coar/resource_type/c_ddb1 +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/DDIInstance +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/datafile +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/dataset +dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/enhancedObjectFile +dnet:publication_resource @=@ 0006 @=@ Diss +dnet:publication_resource @=@ 0006 @=@ Dissertation +dnet:publication_resource @=@ 0006 @=@ Doctoral +dnet:publication_resource @=@ 0006 @=@ DoctoralThesis +dnet:publication_resource @=@ 0006 @=@ PhD thesis +dnet:publication_resource @=@ 0006 @=@ Tesis +dnet:publication_resource @=@ 0006 @=@ Text.Thesis.Doctoral +dnet:publication_resource @=@ 0006 @=@ Theses +dnet:publication_resource @=@ 0006 @=@ Thesis +dnet:publication_resource @=@ 0006 @=@ Thesis or Dissertation +dnet:publication_resource @=@ 0006 @=@ Thesis.Doctoral +dnet:publication_resource @=@ 0006 @=@ doc-type:doctoralThesis +dnet:publication_resource @=@ 0006 @=@ http://purl.org/coar/resource_type/c_db06 +dnet:publication_resource @=@ 0006 @=@ info:eu-repo/semantics/doctoralThesis +dnet:publication_resource @=@ 0006 @=@ publication-thesis +dnet:publication_resource @=@ 0006 @=@ these +dnet:publication_resource @=@ 0006 @=@ these exercice +dnet:publication_resource @=@ 0023 @=@ Event/Event +dnet:publication_resource @=@ 0023 @=@ event +dnet:publication_resource @=@ 0009 @=@ Departmental Technical Report +dnet:publication_resource @=@ 0009 @=@ Informe Técnico +dnet:publication_resource @=@ 0009 @=@ RESEARCH_REPORT +dnet:publication_resource @=@ 0009 @=@ Tech-Report +dnet:publication_resource @=@ 0009 @=@ Technical Report +dnet:publication_resource @=@ 0009 @=@ http://purl.org/coar/resource_type/c_18gh +dnet:publication_resource @=@ 0009 @=@ publication-technicalnote +dnet:publication_resource @=@ 0009 @=@ research report +dnet:publication_resource @=@ 0024 @=@ Video +dnet:publication_resource @=@ 0024 @=@ film +dnet:publication_resource @=@ 0024 @=@ http://purl.org/coar/resource_type/c_12ce +dnet:publication_resource @=@ 0024 @=@ http://purl.org/coar/resource_type/c_8a7e +dnet:publication_resource @=@ 0025 @=@ Diagram +dnet:publication_resource @=@ 0025 @=@ Drawing +dnet:publication_resource @=@ 0025 @=@ Figure +dnet:publication_resource @=@ 0025 @=@ Image/Image +dnet:publication_resource @=@ 0025 @=@ Imagen +dnet:publication_resource @=@ 0025 @=@ Photo +dnet:publication_resource @=@ 0025 @=@ Plot +dnet:publication_resource @=@ 0025 @=@ fotó +dnet:publication_resource @=@ 0025 @=@ grafika +dnet:publication_resource @=@ 0025 @=@ http://purl.org/coar/resource_type/c_ecc8 +dnet:publication_resource @=@ 0025 @=@ image +dnet:publication_resource @=@ 0025 @=@ image-diagram +dnet:publication_resource @=@ 0025 @=@ image-drawing +dnet:publication_resource @=@ 0025 @=@ image-figure +dnet:publication_resource @=@ 0025 @=@ image-other +dnet:publication_resource @=@ 0025 @=@ image-photo +dnet:publication_resource @=@ 0025 @=@ image-plot +dnet:publication_resource @=@ 0026 @=@ http://purl.org/coar/resource_type/c_e9a0 +dnet:publication_resource @=@ 0026 @=@ interactiveResource +dnet:publication_resource @=@ 0011 @=@ Internal note +dnet:publication_resource @=@ 0011 @=@ http://purl.org/coar/resource_type/c_18ww +dnet:publication_resource @=@ 0043 @=@ http://purl.org/coar/resource_type/c_0640 +dnet:publication_resource @=@ 0010 @=@ Inaugural lecture +dnet:publication_resource @=@ 0010 @=@ Material didáctico +dnet:publication_resource @=@ 0010 @=@ Public-Lecture +dnet:publication_resource @=@ 0010 @=@ http://purl.org/coar/resource_type/c_8544 +dnet:publication_resource @=@ 0010 @=@ info:eu-repo/semantics/lecture +dnet:publication_resource @=@ 0010 @=@ lesson +dnet:publication_resource @=@ 0010 @=@ Учебный материал +dnet:publication_resource @=@ 0007 @=@ Diploma Project +dnet:publication_resource @=@ 0007 @=@ MSc Thesis +dnet:publication_resource @=@ 0007 @=@ Master Degree +dnet:publication_resource @=@ 0007 @=@ Master's +dnet:publication_resource @=@ 0007 @=@ Masterarbeit u.a. +dnet:publication_resource @=@ 0007 @=@ Masters (Taught) +dnet:publication_resource @=@ 0007 @=@ Masters thesis +dnet:publication_resource @=@ 0007 @=@ Masters-Thesis.Magister +dnet:publication_resource @=@ 0007 @=@ Tesina +dnet:publication_resource @=@ 0007 @=@ Thesis.Master +dnet:publication_resource @=@ 0007 @=@ Trabajo fin de Máster +dnet:publication_resource @=@ 0007 @=@ doc-type:masterThesis +dnet:publication_resource @=@ 0007 @=@ hdr +dnet:publication_resource @=@ 0007 @=@ http://purl.org/coar/resource_type/c_bdcc +dnet:publication_resource @=@ 0007 @=@ info:eu-repo/semantics/masterThesis +dnet:publication_resource @=@ 0007 @=@ masterThesis +dnet:publication_resource @=@ 0007 @=@ memoire +dnet:publication_resource @=@ 0027 @=@ Model/Model +dnet:publication_resource @=@ 0027 @=@ model +dnet:publication_resource @=@ 0020 @=@ Exhibition +dnet:publication_resource @=@ 0020 @=@ Learning Object +dnet:publication_resource @=@ 0020 @=@ Mapa +dnet:publication_resource @=@ 0020 @=@ Modelo de utilidad +dnet:publication_resource @=@ 0020 @=@ PEDAGOGICAL_DOCUMENT +dnet:publication_resource @=@ 0020 @=@ Partitura +dnet:publication_resource @=@ 0020 @=@ Sitio web +dnet:publication_resource @=@ 0020 @=@ Trabajo de divulgación +dnet:publication_resource @=@ 0020 @=@ Web publication/site +dnet:publication_resource @=@ 0020 @=@ application +dnet:publication_resource @=@ 0020 @=@ artefact +dnet:publication_resource @=@ 0020 @=@ carte +dnet:publication_resource @=@ 0020 @=@ composition +dnet:publication_resource @=@ 0020 @=@ document_audiovisuel +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_12cc +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_12cd +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_1843 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_18cd +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_18cw +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_26e4 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_7ad9 +dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_e059 +dnet:publication_resource @=@ 0020 @=@ info:eu-repo/semantics/other +dnet:publication_resource @=@ 0020 @=@ learningObject +dnet:publication_resource @=@ 0020 @=@ map +dnet:publication_resource @=@ 0020 @=@ misc +dnet:publication_resource @=@ 0020 @=@ other +dnet:publication_resource @=@ 0020 @=@ revue +dnet:publication_resource @=@ 0038 @=@ Abstract +dnet:publication_resource @=@ 0038 @=@ Blog +dnet:publication_resource @=@ 0038 @=@ Book Prospectus +dnet:publication_resource @=@ 0038 @=@ Dictionary Entry +dnet:publication_resource @=@ 0038 @=@ Disclosure +dnet:publication_resource @=@ 0038 @=@ Editorial +dnet:publication_resource @=@ 0038 @=@ Editorial ISI/JCR +dnet:publication_resource @=@ 0038 @=@ Editors +dnet:publication_resource @=@ 0038 @=@ Editors (non peer-reviewed) +dnet:publication_resource @=@ 0038 @=@ Editors (peer-reviewed) +dnet:publication_resource @=@ 0038 @=@ Encyclopedia Entry +dnet:publication_resource @=@ 0038 @=@ Entrada de blog +dnet:publication_resource @=@ 0038 @=@ Funding Submission +dnet:publication_resource @=@ 0038 @=@ HabilitationThesis +dnet:publication_resource @=@ 0038 @=@ License +dnet:publication_resource @=@ 0038 @=@ Manual +dnet:publication_resource @=@ 0038 @=@ Manuscript +dnet:publication_resource @=@ 0038 @=@ Manuscrito +dnet:publication_resource @=@ 0038 @=@ Other publication (non peer-review) +dnet:publication_resource @=@ 0038 @=@ Other publication (peer-review) +dnet:publication_resource @=@ 0038 @=@ Revista +dnet:publication_resource @=@ 0038 @=@ Supervised Student Publication +dnet:publication_resource @=@ 0038 @=@ Tesis/trabajos de grado – Thesis +dnet:publication_resource @=@ 0038 @=@ Text +dnet:publication_resource @=@ 0038 @=@ Text/Text +dnet:publication_resource @=@ 0038 @=@ Trademark +dnet:publication_resource @=@ 0038 @=@ Translation +dnet:publication_resource @=@ 0038 @=@ afterword +dnet:publication_resource @=@ 0038 @=@ avantpropos +dnet:publication_resource @=@ 0038 @=@ bibliography +dnet:publication_resource @=@ 0038 @=@ chronique +dnet:publication_resource @=@ 0038 @=@ compte rendu +dnet:publication_resource @=@ 0038 @=@ correction +dnet:publication_resource @=@ 0038 @=@ foreword +dnet:publication_resource @=@ 0038 @=@ habilitation à diriger des recherches +dnet:publication_resource @=@ 0038 @=@ historicalDocument +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_0040 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_0857 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_18cf +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_18wz +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_3e5a +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_46ec +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_6947 +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_7acd +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_86bc +dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_b239 +dnet:publication_resource @=@ 0038 @=@ note de lecture +dnet:publication_resource @=@ 0038 @=@ notedelecture +dnet:publication_resource @=@ 0038 @=@ other publication +dnet:publication_resource @=@ 0038 @=@ postface +dnet:publication_resource @=@ 0038 @=@ publication-other +dnet:publication_resource @=@ 0038 @=@ revuedepresse +dnet:publication_resource @=@ 0038 @=@ sa_component +dnet:publication_resource @=@ 0038 @=@ standard +dnet:publication_resource @=@ 0038 @=@ standard-series +dnet:publication_resource @=@ 0013 @=@ A3 Kirjan tai muun kokoomateoksen osa +dnet:publication_resource @=@ 0013 @=@ Book Part (author) +dnet:publication_resource @=@ 0013 @=@ Book Section / Chapter +dnet:publication_resource @=@ 0013 @=@ Book chapter or Essay in book +dnet:publication_resource @=@ 0013 @=@ Book editorial +dnet:publication_resource @=@ 0013 @=@ Book section +dnet:publication_resource @=@ 0013 @=@ Book_Chapter +dnet:publication_resource @=@ 0013 @=@ Buchbeitrag +dnet:publication_resource @=@ 0013 @=@ Capítulo de libro +dnet:publication_resource @=@ 0013 @=@ Contribution to International Book/Monograph +dnet:publication_resource @=@ 0013 @=@ Contribution to International Book/Monograph ISI/JCR +dnet:publication_resource @=@ 0013 @=@ Contribution to National Book/Monograph +dnet:publication_resource @=@ 0013 @=@ Contribution to book (non peer-reviewed) +dnet:publication_resource @=@ 0013 @=@ Contribution to book (peer-reviewed) +dnet:publication_resource @=@ 0013 @=@ Part of book - chapter +dnet:publication_resource @=@ 0013 @=@ book chapter +dnet:publication_resource @=@ 0013 @=@ book-part +dnet:publication_resource @=@ 0013 @=@ bookPart +dnet:publication_resource @=@ 0013 @=@ book_content +dnet:publication_resource @=@ 0013 @=@ chapitre_ouvrage +dnet:publication_resource @=@ 0013 @=@ chapter +dnet:publication_resource @=@ 0013 @=@ doc-type:bookPart +dnet:publication_resource @=@ 0013 @=@ http://purl.org/coar/resource_type/c_3248 +dnet:publication_resource @=@ 0013 @=@ info:eu-repo/semantics/bookPart +dnet:publication_resource @=@ 0013 @=@ publication-section +dnet:publication_resource @=@ 0013 @=@ reference-entry +dnet:publication_resource @=@ 0013 @=@ reference_entry +dnet:publication_resource @=@ 0013 @=@ scientific book chapter +dnet:publication_resource @=@ 0013 @=@ Глава монографии +dnet:publication_resource @=@ 0019 @=@ H1 Myönnetty patentti +dnet:publication_resource @=@ 0019 @=@ Patent +dnet:publication_resource @=@ 0019 @=@ Patente +dnet:publication_resource @=@ 0019 @=@ Solicitud de patente +dnet:publication_resource @=@ 0019 @=@ Traducción de patente +dnet:publication_resource @=@ 0019 @=@ brevet +dnet:publication_resource @=@ 0019 @=@ http://purl.org/coar/resource_type/c_15cd +dnet:publication_resource @=@ 0019 @=@ info:eu-repo/semantics/patent +dnet:publication_resource @=@ 0019 @=@ publication-patent +dnet:publication_resource @=@ 0028 @=@ Service +dnet:publication_resource @=@ 0028 @=@ physicalObject +dnet:publication_resource @=@ 0016 @=@ Pre Print +dnet:publication_resource @=@ 0016 @=@ Pre-print +dnet:publication_resource @=@ 0016 @=@ http://purl.org/coar/resource_type/c_816b +dnet:publication_resource @=@ 0016 @=@ info:eu-repo/semantics/preprint +dnet:publication_resource @=@ 0016 @=@ publication-preprint +dnet:publication_resource @=@ 0016 @=@ Препринт +dnet:publication_resource @=@ 0034 @=@ Project deliverable +dnet:publication_resource @=@ 0034 @=@ http://purl.org/coar/resource_type/c_18op +dnet:publication_resource @=@ 0034 @=@ publication-deliverable +dnet:publication_resource @=@ 0035 @=@ Project milestone +dnet:publication_resource @=@ 0035 @=@ publication-milestone +dnet:publication_resource @=@ 0036 @=@ Proposal +dnet:publication_resource @=@ 0036 @=@ http://purl.org/coar/resource_type/c_baaf +dnet:publication_resource @=@ 0036 @=@ research-proposal +dnet:publication_resource @=@ 0017 @=@ ACTIVITY_REPORT +dnet:publication_resource @=@ 0017 @=@ Commissioned report +dnet:publication_resource @=@ 0017 @=@ D4 Julkaistu kehittämis- tai tutkimusraportti tai -selvitys +dnet:publication_resource @=@ 0017 @=@ Deliverable +dnet:publication_resource @=@ 0017 @=@ Documento tecnico +dnet:publication_resource @=@ 0017 @=@ Project Report +dnet:publication_resource @=@ 0017 @=@ Software documentation +dnet:publication_resource @=@ 0017 @=@ brief-report +dnet:publication_resource @=@ 0017 @=@ case-report +dnet:publication_resource @=@ 0017 @=@ chapitre_rapport +dnet:publication_resource @=@ 0017 @=@ doc-type:report +dnet:publication_resource @=@ 0017 @=@ document_institutionnel +dnet:publication_resource @=@ 0017 @=@ document_technique +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_186u +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18hj +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18wq +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18ws +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_71bd +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_93fc +dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_ba1f +dnet:publication_resource @=@ 0017 @=@ info:eu-repo/semantics/report +dnet:publication_resource @=@ 0017 @=@ publication-report +dnet:publication_resource @=@ 0017 @=@ publication-softwaredocumentation +dnet:publication_resource @=@ 0017 @=@ rapport_expertise +dnet:publication_resource @=@ 0017 @=@ rapport_mission +dnet:publication_resource @=@ 0017 @=@ report +dnet:publication_resource @=@ 0017 @=@ report-paper +dnet:publication_resource @=@ 0017 @=@ report-paper_title +dnet:publication_resource @=@ 0017 @=@ report-series +dnet:publication_resource @=@ 0017 @=@ support_cours +dnet:publication_resource @=@ 0014 @=@ Arbeitspapier +dnet:publication_resource @=@ 0014 @=@ Departmental Bulletin Paper +dnet:publication_resource @=@ 0014 @=@ Documento de trabajo +dnet:publication_resource @=@ 0014 @=@ Paper +dnet:publication_resource @=@ 0014 @=@ Project description +dnet:publication_resource @=@ 0014 @=@ Research-Paper +dnet:publication_resource @=@ 0014 @=@ ResearchPaper +dnet:publication_resource @=@ 0014 @=@ Working / discussion paper +dnet:publication_resource @=@ 0014 @=@ Working Paper +dnet:publication_resource @=@ 0014 @=@ Working Paper / Technical Report +dnet:publication_resource @=@ 0014 @=@ doc-type:workingPaper +dnet:publication_resource @=@ 0014 @=@ http://purl.org/coar/resource_type/c_8042 +dnet:publication_resource @=@ 0014 @=@ info:eu-repo/semantics/paper +dnet:publication_resource @=@ 0014 @=@ info:eu-repo/semantics/workingPaper +dnet:publication_resource @=@ 0014 @=@ publication-workingpaper +dnet:publication_resource @=@ 0014 @=@ workingPaper +dnet:publication_resource @=@ 0015 @=@ A2 Katsausartikkeli tieteellisessä aikakauslehdessä +dnet:publication_resource @=@ 0015 @=@ Book Review +dnet:publication_resource @=@ 0015 @=@ Book/Film/Article review +dnet:publication_resource @=@ 0015 @=@ Literature review +dnet:publication_resource @=@ 0015 @=@ Peer review +dnet:publication_resource @=@ 0015 @=@ Reseña bibliográfica +dnet:publication_resource @=@ 0015 @=@ Review Article +dnet:publication_resource @=@ 0015 @=@ RezensionReview +dnet:publication_resource @=@ 0015 @=@ book-review +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_ba08 +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_dcae04bc +dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_efa0 +dnet:publication_resource @=@ 0015 @=@ info:eu-repo/semantics/review +dnet:publication_resource @=@ 0015 @=@ peer-review +dnet:publication_resource @=@ 0029 @=@ Software +dnet:publication_resource @=@ 0029 @=@ Software/Software +dnet:publication_resource @=@ 0029 @=@ Workflow +dnet:publication_resource @=@ 0029 @=@ Workflow/Workflow +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_393c +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_5ce6 +dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_c950 +dnet:publication_resource @=@ 0032 @=@ http://purl.org/coar/resource_type/c_7bab +dnet:publication_resource @=@ 0030 @=@ http://purl.org/coar/resource_type/c_18cc +dnet:publication_resource @=@ 0030 @=@ sound +dnet:publication_resource @=@ 0044 @=@ Graduate diploma +dnet:publication_resource @=@ 0044 @=@ Undergraduate diploma +dnet:publication_resource @=@ 0000 @=@ UNKNOWN +dnet:publication_resource @=@ 0042 @=@ EGI Virtual Appliance dnet:languages @=@ abk @=@ ab dnet:languages @=@ aar @=@ aa dnet:languages @=@ afr @=@ af @@ -317,24 +539,37 @@ dnet:languages @=@ eng @=@ en_gb dnet:languages @=@ eng @=@ en_nz dnet:languages @=@ eng @=@ en_us dnet:languages @=@ eng @=@ english +dnet:languages @=@ eng @=@ en-us +dnet:languages @=@ eng @=@ en-US +dnet:languages @=@ eng @=@ English +dnet:languages @=@ eng @=@ EN +dnet:languages @=@ eng @=@ en angielski +dnet:languages @=@ eng @=@ en-GB +dnet:languages @=@ eng @=@ Englisch dnet:languages @=@ epo @=@ eo dnet:languages @=@ est @=@ et dnet:languages @=@ fao @=@ fo dnet:languages @=@ fij @=@ fj dnet:languages @=@ fin @=@ fi +dnet:languages @=@ fin @=@ Finnish dnet:languages @=@ fra/fre @=@ fr +dnet:languages @=@ fra/fre @=@ FR dnet:languages @=@ fra/fre @=@ fr_be dnet:languages @=@ fra/fre @=@ fr_fr dnet:languages @=@ fra/fre @=@ fre/fra +dnet:languages @=@ fra/fre @=@ fra dnet:languages @=@ fry @=@ fy dnet:languages @=@ glg @=@ gl dnet:languages @=@ geo/kat @=@ ka dnet:languages @=@ deu/ger @=@ de dnet:languages @=@ deu/ger @=@ ger/deu dnet:languages @=@ deu/ger @=@ german -dnet:languages @=@ deu/ger @=@ de_DE +dnet:languages @=@ deu/ger @=@ ger +dnet:languages @=@ deu/ger @=@ deu +dnet:languages @=@ deu/ger @=@ DE-de dnet:languages @=@ ell/gre @=@ el dnet:languages @=@ ell/gre @=@ gr +dnet:languages @=@ ell/gre @=@ el-GR dnet:languages @=@ kal @=@ kl dnet:languages @=@ grn @=@ gn dnet:languages @=@ guj @=@ gu @@ -418,6 +653,7 @@ dnet:languages @=@ tam @=@ ta dnet:languages @=@ tat @=@ tt dnet:languages @=@ tel @=@ te dnet:languages @=@ tha @=@ th +dnet:languages @=@ tha @=@ thai dnet:languages @=@ bod/tib @=@ bo dnet:languages @=@ tir @=@ ti dnet:languages @=@ tog @=@ to @@ -428,6 +664,8 @@ dnet:languages @=@ tuk @=@ tk dnet:languages @=@ twi @=@ tw dnet:languages @=@ uig @=@ ug dnet:languages @=@ ukr @=@ uk +dnet:languages @=@ und @=@ UNKNOWN +dnet:languages @=@ und @=@ none dnet:languages @=@ urd @=@ ur dnet:languages @=@ uzb @=@ uz dnet:languages @=@ vie @=@ vi @@ -440,566 +678,280 @@ dnet:languages @=@ yid @=@ yi dnet:languages @=@ yor @=@ yo dnet:languages @=@ zha @=@ za dnet:languages @=@ zul @=@ zu -dnet:pid_types @=@ orcid @=@ ORCID12 -dnet:subject_classification_typologies @=@ jel @=@ jelElement +dnet:result_typologies @=@ dataset @=@ 0021 +dnet:result_typologies @=@ dataset @=@ 0024 +dnet:result_typologies @=@ dataset @=@ 0025 +dnet:result_typologies @=@ dataset @=@ 0030 +dnet:result_typologies @=@ dataset @=@ 0033 +dnet:result_typologies @=@ dataset @=@ 0037 +dnet:result_typologies @=@ dataset @=@ 0039 +dnet:result_typologies @=@ dataset @=@ 0046 +dnet:result_typologies @=@ other @=@ 0000 +dnet:result_typologies @=@ other @=@ 0010 +dnet:result_typologies @=@ other @=@ 0018 +dnet:result_typologies @=@ other @=@ 0020 +dnet:result_typologies @=@ other @=@ 0022 +dnet:result_typologies @=@ other @=@ 0023 +dnet:result_typologies @=@ other @=@ 0026 +dnet:result_typologies @=@ other @=@ 0027 +dnet:result_typologies @=@ other @=@ 0028 +dnet:result_typologies @=@ other @=@ 0042 +dnet:result_typologies @=@ publication @=@ 0001 +dnet:result_typologies @=@ publication @=@ 0002 +dnet:result_typologies @=@ publication @=@ 0004 +dnet:result_typologies @=@ publication @=@ 0005 +dnet:result_typologies @=@ publication @=@ 0006 +dnet:result_typologies @=@ publication @=@ 0007 +dnet:result_typologies @=@ publication @=@ 0008 +dnet:result_typologies @=@ publication @=@ 0009 +dnet:result_typologies @=@ publication @=@ 0011 +dnet:result_typologies @=@ publication @=@ 0012 +dnet:result_typologies @=@ publication @=@ 0013 +dnet:result_typologies @=@ publication @=@ 0014 +dnet:result_typologies @=@ publication @=@ 0015 +dnet:result_typologies @=@ publication @=@ 0016 +dnet:result_typologies @=@ publication @=@ 0017 +dnet:result_typologies @=@ publication @=@ 0019 +dnet:result_typologies @=@ publication @=@ 0031 +dnet:result_typologies @=@ publication @=@ 0032 +dnet:result_typologies @=@ publication @=@ 0034 +dnet:result_typologies @=@ publication @=@ 0035 +dnet:result_typologies @=@ publication @=@ 0036 +dnet:result_typologies @=@ publication @=@ 0038 +dnet:result_typologies @=@ publication @=@ 0044 +dnet:result_typologies @=@ publication @=@ 0045 +dnet:result_typologies @=@ software @=@ 0029 +dnet:result_typologies @=@ software @=@ 0040 +dnet:countries @=@ AF @=@ AFG +dnet:countries @=@ AF @=@ Afghanistan +dnet:countries @=@ AD @=@ Andorra +dnet:countries @=@ AO @=@ Angola +dnet:countries @=@ AR @=@ ARG +dnet:countries @=@ AR @=@ Argentina +dnet:countries @=@ AU @=@ AUS +dnet:countries @=@ AU @=@ Australia +dnet:countries @=@ AT @=@ AUT +dnet:countries @=@ AT @=@ Austria +dnet:countries @=@ AZ @=@ AZE +dnet:countries @=@ BD @=@ Bangladesh +dnet:countries @=@ BY @=@ Belarus +dnet:countries @=@ BE @=@ BEL +dnet:countries @=@ BE @=@ Belgium +dnet:countries @=@ BJ @=@ BEN +dnet:countries @=@ BO @=@ Bolivia, Plurinational State of +dnet:countries @=@ BA @=@ BIH +dnet:countries @=@ BA @=@ Bosnia-Hercegovina +dnet:countries @=@ BR @=@ BRA +dnet:countries @=@ BR @=@ Brazil +dnet:countries @=@ BG @=@ Bulgaria +dnet:countries @=@ BF @=@ BFA +dnet:countries @=@ KH @=@ Cambodia +dnet:countries @=@ KH @=@ Cambogia +dnet:countries @=@ KH @=@ Campuchea +dnet:countries @=@ CM @=@ CMR +dnet:countries @=@ CA @=@ CAN +dnet:countries @=@ CA @=@ Canada +dnet:countries @=@ CV @=@ Cape Verde +dnet:countries @=@ CL @=@ CHL +dnet:countries @=@ CL @=@ Chile +dnet:countries @=@ CN @=@ CHN +dnet:countries @=@ CN @=@ China +dnet:countries @=@ CO @=@ COL +dnet:countries @=@ CO @=@ Colombia +dnet:countries @=@ CD @=@ Congo +dnet:countries @=@ CD @=@ Congo Democratic Republic (formerly Zaire) +dnet:countries @=@ CD @=@ Congo, Republic +dnet:countries @=@ CD @=@ Congo, the Democratic Republic of the +dnet:countries @=@ CD @=@ Zaire +dnet:countries @=@ CR @=@ CRI +dnet:countries @=@ CI @=@ CIV +dnet:countries @=@ CI @=@ Ivory Coast +dnet:countries @=@ HR @=@ Croatia +dnet:countries @=@ HR @=@ HRV +dnet:countries @=@ CY @=@ CYP +dnet:countries @=@ CY @=@ Cyprus +dnet:countries @=@ CZ @=@ CZE +dnet:countries @=@ CZ @=@ Czech Republic +dnet:countries @=@ CZ @=@ Czechia +dnet:countries @=@ CZ @=@ Czechoslovakia +dnet:countries @=@ DK @=@ DNK +dnet:countries @=@ DK @=@ Denmark +dnet:countries @=@ EC @=@ Ecuador +dnet:countries @=@ EG @=@ EGY +dnet:countries @=@ EG @=@ Egypt +dnet:countries @=@ SV @=@ SLV +dnet:countries @=@ EE @=@ EST +dnet:countries @=@ EE @=@ Estonia +dnet:countries @=@ ET @=@ ETH +dnet:countries @=@ EU @=@ EEC +dnet:countries @=@ FJ @=@ FJI +dnet:countries @=@ FI @=@ FIN +dnet:countries @=@ FI @=@ Finland +dnet:countries @=@ MK @=@ Macedonia +dnet:countries @=@ MK @=@ Macedonia, the Former Yugoslav Republic Of +dnet:countries @=@ MK @=@ North Macedonia +dnet:countries @=@ FR @=@ FRA +dnet:countries @=@ FR @=@ France +dnet:countries @=@ PF @=@ French Polynesia +dnet:countries @=@ PF @=@ PYF +dnet:countries @=@ TF @=@ French Southern Territories +dnet:countries @=@ GE @=@ Georgia +dnet:countries @=@ DE @=@ DEU +dnet:countries @=@ DE @=@ Germany +dnet:countries @=@ DE @=@ Germany, Berlin +dnet:countries @=@ GH @=@ GHA +dnet:countries @=@ GR @=@ EL +dnet:countries @=@ GR @=@ GRC +dnet:countries @=@ GL @=@ GRL +dnet:countries @=@ GN @=@ Guinea +dnet:countries @=@ GW @=@ Guinea-Bissau +dnet:countries @=@ VA @=@ Vatican State +dnet:countries @=@ HK @=@ HKG +dnet:countries @=@ HK @=@ Hong Kong +dnet:countries @=@ HK @=@ Hongkong +dnet:countries @=@ HU @=@ HUN +dnet:countries @=@ HU @=@ Hungary +dnet:countries @=@ IS @=@ ISL +dnet:countries @=@ IN @=@ IND +dnet:countries @=@ IN @=@ India +dnet:countries @=@ ID @=@ IDN +dnet:countries @=@ ID @=@ Indonesia +dnet:countries @=@ IR @=@ Iran +dnet:countries @=@ IR @=@ Iran, Islamic Republic of +dnet:countries @=@ IE @=@ IRL +dnet:countries @=@ IE @=@ Ireland +dnet:countries @=@ IL @=@ ISR +dnet:countries @=@ IL @=@ Israel +dnet:countries @=@ IT @=@ ITA +dnet:countries @=@ IT @=@ Italy +dnet:countries @=@ JM @=@ Jamaica +dnet:countries @=@ JP @=@ JPN +dnet:countries @=@ JP @=@ Japan +dnet:countries @=@ KZ @=@ KAZ +dnet:countries @=@ KZ @=@ Kazakistan +dnet:countries @=@ KZ @=@ Kazakstan +dnet:countries @=@ KE @=@ KEN +dnet:countries @=@ KE @=@ Kenya +dnet:countries @=@ KR @=@ KOR +dnet:countries @=@ KR @=@ Korea, Republic of +dnet:countries @=@ KR @=@ Korean Republic (South Korea) +dnet:countries @=@ KP @=@ PRK +dnet:countries @=@ LV @=@ LVA +dnet:countries @=@ LY @=@ Libya +dnet:countries @=@ LT @=@ LTU +dnet:countries @=@ LU @=@ LUX +dnet:countries @=@ LU @=@ Luxembourg +dnet:countries @=@ MO @=@ Macao +dnet:countries @=@ MG @=@ Madagascar +dnet:countries @=@ MY @=@ Malaysia +dnet:countries @=@ ML @=@ Mali +dnet:countries @=@ MT @=@ Malta +dnet:countries @=@ MU @=@ Mauritius +dnet:countries @=@ MX @=@ MEX +dnet:countries @=@ MX @=@ Mexico +dnet:countries @=@ FM @=@ Micronesia +dnet:countries @=@ MD @=@ Moldova +dnet:countries @=@ MD @=@ Moldova, Republic of +dnet:countries @=@ MN @=@ Mongolia +dnet:countries @=@ MA @=@ Morocco +dnet:countries @=@ MZ @=@ Mozambique +dnet:countries @=@ NA @=@ NAM +dnet:countries @=@ NL @=@ NLD +dnet:countries @=@ NL @=@ Netherlands +dnet:countries @=@ AN @=@ Netherlands Antilles +dnet:countries @=@ NC @=@ NCL +dnet:countries @=@ NZ @=@ NZL +dnet:countries @=@ NZ @=@ New Zealand +dnet:countries @=@ NO @=@ NOR +dnet:countries @=@ NO @=@ Norway +dnet:countries @=@ OC @=@ Australasia +dnet:countries @=@ OM @=@ Oman +dnet:countries @=@ PK @=@ PAK +dnet:countries @=@ PK @=@ Pakistan +dnet:countries @=@ PS @=@ Palestin, State of +dnet:countries @=@ PS @=@ Palestine, State of +dnet:countries @=@ PS @=@ Palestinian Territory, Occupied +dnet:countries @=@ PA @=@ PAN +dnet:countries @=@ PA @=@ Panama +dnet:countries @=@ PG @=@ PapuaNew Guinea +dnet:countries @=@ PE @=@ PER +dnet:countries @=@ PH @=@ PHL +dnet:countries @=@ PH @=@ Philippines +dnet:countries @=@ PL @=@ POL +dnet:countries @=@ PL @=@ Poland +dnet:countries @=@ PT @=@ PRT +dnet:countries @=@ PT @=@ Portugal +dnet:countries @=@ PR @=@ Puerto Rico +dnet:countries @=@ RO @=@ ROU +dnet:countries @=@ RO @=@ Romania +dnet:countries @=@ RU @=@ RUS +dnet:countries @=@ RU @=@ Russia +dnet:countries @=@ RU @=@ Russian Federation +dnet:countries @=@ RE @=@ Réunion +dnet:countries @=@ KN @=@ Saint Kitts And Nevis +dnet:countries @=@ SA @=@ Saudi Arabia +dnet:countries @=@ SN @=@ SEN +dnet:countries @=@ RS @=@ SRB +dnet:countries @=@ CS @=@ Serbia and Montenegro +dnet:countries @=@ SG @=@ SGP +dnet:countries @=@ SG @=@ Singapore +dnet:countries @=@ SK @=@ SVK +dnet:countries @=@ SI @=@ SVN +dnet:countries @=@ SI @=@ Slovenia +dnet:countries @=@ ZA @=@ South Africa +dnet:countries @=@ ZA @=@ ZAF +dnet:countries @=@ ES @=@ ESP +dnet:countries @=@ ES @=@ Spain +dnet:countries @=@ LK @=@ LKA +dnet:countries @=@ LK @=@ Sri Lanka +dnet:countries @=@ SD @=@ SDN +dnet:countries @=@ SR @=@ Suriname +dnet:countries @=@ SE @=@ SWE +dnet:countries @=@ SE @=@ Sweden +dnet:countries @=@ CH @=@ CHE +dnet:countries @=@ CH @=@ Switzerland +dnet:countries @=@ SY @=@ Syria +dnet:countries @=@ ST @=@ Sao Tome and Principe +dnet:countries @=@ TW @=@ TWN +dnet:countries @=@ TW @=@ Taiwan +dnet:countries @=@ TW @=@ Taiwan, Province of China +dnet:countries @=@ TZ @=@ Tanzania +dnet:countries @=@ TZ @=@ Tanzania, United Republic of +dnet:countries @=@ TH @=@ THA +dnet:countries @=@ TH @=@ Thailand +dnet:countries @=@ TL @=@ East Timor +dnet:countries @=@ TN @=@ TUN +dnet:countries @=@ TN @=@ Tunisia +dnet:countries @=@ TR @=@ TUR +dnet:countries @=@ TR @=@ Turkey +dnet:countries @=@ UNKNOWN @=@ AAA +dnet:countries @=@ UNKNOWN @=@ [Unknown] +dnet:countries @=@ UNKNOWN @=@ _? +dnet:countries @=@ UA @=@ UKR +dnet:countries @=@ UA @=@ Ukraine +dnet:countries @=@ AE @=@ United Arab Emirates +dnet:countries @=@ GB @=@ England +dnet:countries @=@ GB @=@ GBR +dnet:countries @=@ GB @=@ Great Britain +dnet:countries @=@ GB @=@ Great Britain and Northern Ireland +dnet:countries @=@ GB @=@ Scotland +dnet:countries @=@ GB @=@ UK +dnet:countries @=@ GB @=@ United Kingdom +dnet:countries @=@ US @=@ USA +dnet:countries @=@ US @=@ United States +dnet:countries @=@ US @=@ United States of America +dnet:countries @=@ UY @=@ Uruguay +dnet:countries @=@ UZ @=@ Uzbekistan +dnet:countries @=@ VE @=@ Venezuela, Bolivarian Republic of +dnet:countries @=@ VN @=@ Vietnam +dnet:countries @=@ VG @=@ British Virgin Islands +dnet:countries @=@ YU @=@ Jugoslavia +dnet:countries @=@ YU @=@ Yugoslavia +dnet:countries @=@ ZW @=@ ABW dnet:protocols @=@ oai @=@ OAI-PMH dnet:protocols @=@ oai @=@ OAI_PMH -dnet:publication_resource @=@ 0018 @=@ Comentario -dnet:publication_resource @=@ 0018 @=@ Comment/debate -dnet:publication_resource @=@ 0018 @=@ http://purl.org/coar/resource_type/c_1162 -dnet:publication_resource @=@ 0018 @=@ info:eu-repo/semantics/annotation -dnet:publication_resource @=@ 0001 @=@ A1 Alkuperäisartikkeli tieteellisessä aikakauslehdessä -dnet:publication_resource @=@ 0001 @=@ Article -dnet:publication_resource @=@ 0001 @=@ Article (author) -dnet:publication_resource @=@ 0001 @=@ Article - letter to the editor -dnet:publication_resource @=@ 0001 @=@ Article / Letter to editor -dnet:publication_resource @=@ 0001 @=@ Article / Letter to the editor -dnet:publication_resource @=@ 0001 @=@ Article / Newspaper -dnet:publication_resource @=@ 0001 @=@ Article in journal -dnet:publication_resource @=@ 0001 @=@ Article in monograph or in proceedings -dnet:publication_resource @=@ 0001 @=@ Article in proceedings -dnet:publication_resource @=@ 0001 @=@ Article-letter to the editor -dnet:publication_resource @=@ 0001 @=@ Article/Letter to editor -dnet:publication_resource @=@ 0001 @=@ Articles in Journals -dnet:publication_resource @=@ 0001 @=@ Articolo -dnet:publication_resource @=@ 0001 @=@ Articulo -dnet:publication_resource @=@ 0001 @=@ Artículo -dnet:publication_resource @=@ 0001 @=@ Artículo - Article -dnet:publication_resource @=@ 0001 @=@ Aufsatz -dnet:publication_resource @=@ 0001 @=@ Clinical Study -dnet:publication_resource @=@ 0001 @=@ Institutional Series -dnet:publication_resource @=@ 0001 @=@ International Journal -dnet:publication_resource @=@ 0001 @=@ International Journal Abstract -dnet:publication_resource @=@ 0001 @=@ International Journal ISI/JCR -dnet:publication_resource @=@ 0001 @=@ Journal (full / special issue) -dnet:publication_resource @=@ 0001 @=@ Journal Article/Review -dnet:publication_resource @=@ 0001 @=@ Journal article -dnet:publication_resource @=@ 0001 @=@ Journal article (on-line or printed) -dnet:publication_resource @=@ 0001 @=@ Journal articles -dnet:publication_resource @=@ 0001 @=@ Journal paper -dnet:publication_resource @=@ 0001 @=@ Makale -dnet:publication_resource @=@ 0001 @=@ National Journal -dnet:publication_resource @=@ 0001 @=@ Original article (non peer-reviewed) -dnet:publication_resource @=@ 0001 @=@ Original article (peer-reviewed) -dnet:publication_resource @=@ 0001 @=@ Peer-reviewed Article -dnet:publication_resource @=@ 0001 @=@ Publication - Article -dnet:publication_resource @=@ 0001 @=@ Published Journal Article -dnet:publication_resource @=@ 0001 @=@ Research Article -dnet:publication_resource @=@ 0001 @=@ Review article (non peer-reviewed) -dnet:publication_resource @=@ 0001 @=@ Review article (peer-reviewed) -dnet:publication_resource @=@ 0001 @=@ Straipsnis / Article -dnet:publication_resource @=@ 0001 @=@ Volumes Edited / Special Issues -dnet:publication_resource @=@ 0001 @=@ art -dnet:publication_resource @=@ 0001 @=@ article in non peer-reviewed journal -dnet:publication_resource @=@ 0001 @=@ article in peer-reviewed journal -dnet:publication_resource @=@ 0001 @=@ article-commentary -dnet:publication_resource @=@ 0001 @=@ article_site_web -dnet:publication_resource @=@ 0001 @=@ artykuł -dnet:publication_resource @=@ 0001 @=@ artículo de revisión -dnet:publication_resource @=@ 0001 @=@ doc-type:Journal Article -dnet:publication_resource @=@ 0001 @=@ doc-type:article -dnet:publication_resource @=@ 0001 @=@ foly -dnet:publication_resource @=@ 0001 @=@ folyóiratcikk -dnet:publication_resource @=@ 0001 @=@ folyoiratcikk -dnet:publication_resource @=@ 0001 @=@ folyóiratcikk -dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 -dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_545b -dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_6501 -dnet:publication_resource @=@ 0001 @=@ http://purl.org/coar/resource_type/c_7877 -dnet:publication_resource @=@ 0001 @=@ http://purl.org/escidoc/metadata/ves/publication-types/article -dnet:publication_resource @=@ 0001 @=@ in-brief -dnet:publication_resource @=@ 0001 @=@ info:eu-repo/semantics/Journal Article -dnet:publication_resource @=@ 0001 @=@ info:eu-repo/semantics/article -dnet:publication_resource @=@ 0001 @=@ journal-article -dnet:publication_resource @=@ 0001 @=@ journalArticle -dnet:publication_resource @=@ 0001 @=@ journal_article -dnet:publication_resource @=@ 0001 @=@ letter -dnet:publication_resource @=@ 0001 @=@ non peer-reviewed article -dnet:publication_resource @=@ 0001 @=@ partial-retraction -dnet:publication_resource @=@ 0001 @=@ proceeding with peer review -dnet:publication_resource @=@ 0001 @=@ rapid-communication -dnet:publication_resource @=@ 0001 @=@ reply -dnet:publication_resource @=@ 0001 @=@ research-article -dnet:publication_resource @=@ 0001 @=@ retraction -dnet:publication_resource @=@ 0001 @=@ review-article -dnet:publication_resource @=@ 0001 @=@ text (article) -dnet:publication_resource @=@ 0001 @=@ Рецензована стаття -dnet:publication_resource @=@ 0001 @=@ Статья -dnet:publication_resource @=@ 0033 @=@ AUDIOVISUAL_DOCUMENT -dnet:publication_resource @=@ 0033 @=@ Audiovisual/Audiovisual -dnet:publication_resource @=@ 0033 @=@ http://purl.org/coar/resource_type/c_c513 -dnet:publication_resource @=@ 0008 @=@ Bachelor's -dnet:publication_resource @=@ 0008 @=@ Bachelor's Degree -dnet:publication_resource @=@ 0008 @=@ Bachelors Thesis -dnet:publication_resource @=@ 0008 @=@ Proyecto fin de carrera -dnet:publication_resource @=@ 0008 @=@ Undergraduate Thesis -dnet:publication_resource @=@ 0008 @=@ http://purl.org/coar/resource_type/c_7a1f -dnet:publication_resource @=@ 0008 @=@ info:eu-repo/semantics/bachelorThesis -dnet:publication_resource @=@ 0008 @=@ выпускная бакалаврская работа -dnet:publication_resource @=@ 0002 @=@ Book (monograph) -dnet:publication_resource @=@ 0002 @=@ Book (non peer-reviewed) -dnet:publication_resource @=@ 0002 @=@ Book (peer-reviewed) -dnet:publication_resource @=@ 0002 @=@ Book - monograph - editorial book -dnet:publication_resource @=@ 0002 @=@ Book Section -dnet:publication_resource @=@ 0002 @=@ Book as author -dnet:publication_resource @=@ 0002 @=@ Books -dnet:publication_resource @=@ 0002 @=@ Buch -dnet:publication_resource @=@ 0002 @=@ Capítulo de Libro - Book Section -dnet:publication_resource @=@ 0002 @=@ Edited Book -dnet:publication_resource @=@ 0002 @=@ International Book/Monograph -dnet:publication_resource @=@ 0002 @=@ Knyga / Book -dnet:publication_resource @=@ 0002 @=@ Książka -dnet:publication_resource @=@ 0002 @=@ Libro -dnet:publication_resource @=@ 0002 @=@ Libro - Book -dnet:publication_resource @=@ 0002 @=@ Monografia -dnet:publication_resource @=@ 0002 @=@ Monograph -dnet:publication_resource @=@ 0002 @=@ National Book/Monograph -dnet:publication_resource @=@ 0002 @=@ atlas -dnet:publication_resource @=@ 0002 @=@ book -dnet:publication_resource @=@ 0002 @=@ book-series -dnet:publication_resource @=@ 0002 @=@ book-set -dnet:publication_resource @=@ 0002 @=@ book-track -dnet:publication_resource @=@ 0002 @=@ book_series -dnet:publication_resource @=@ 0002 @=@ book_title -dnet:publication_resource @=@ 0002 @=@ doc-type:book -dnet:publication_resource @=@ 0002 @=@ eBook -dnet:publication_resource @=@ 0002 @=@ edited-book -dnet:publication_resource @=@ 0002 @=@ http://purl.org/coar/resource_type/c_2f33 -dnet:publication_resource @=@ 0002 @=@ http://purl.org/eprint/type/Book -dnet:publication_resource @=@ 0002 @=@ info:eu-repo/semantics/book -dnet:publication_resource @=@ 0002 @=@ könyv -dnet:publication_resource @=@ 0002 @=@ reference-book -dnet:publication_resource @=@ 0002 @=@ scientific book -dnet:publication_resource @=@ 0002 @=@ Книга -dnet:publication_resource @=@ 0002 @=@ Монография -dnet:publication_resource @=@ 0002 @=@ Учебник -dnet:publication_resource @=@ 0037 @=@ clinicalTrial -dnet:publication_resource @=@ 0037 @=@ http://purl.org/coar/resource_type/c_cb28 -dnet:publication_resource @=@ 0022 @=@ collection -dnet:publication_resource @=@ 0004 @=@ A4 Artikkeli konferenssijulkaisussa -dnet:publication_resource @=@ 0004 @=@ Article from Conference in a Journal -dnet:publication_resource @=@ 0004 @=@ Comunicación Congreso -dnet:publication_resource @=@ 0004 @=@ Comunicación de congreso -dnet:publication_resource @=@ 0004 @=@ Conference Abstract -dnet:publication_resource @=@ 0004 @=@ Conference Paper -dnet:publication_resource @=@ 0004 @=@ Conference Poster -dnet:publication_resource @=@ 0004 @=@ Conference Proceedings -dnet:publication_resource @=@ 0004 @=@ Conference Program -dnet:publication_resource @=@ 0004 @=@ Conference article -dnet:publication_resource @=@ 0004 @=@ Conference contribution -dnet:publication_resource @=@ 0004 @=@ Conference lecture -dnet:publication_resource @=@ 0004 @=@ Conference or Workshop Item -dnet:publication_resource @=@ 0004 @=@ Conference paper, poster, etc. -dnet:publication_resource @=@ 0004 @=@ Conference paper/abstract -dnet:publication_resource @=@ 0004 @=@ Conference papers -dnet:publication_resource @=@ 0004 @=@ Conference preprint -dnet:publication_resource @=@ 0004 @=@ Conference report -dnet:publication_resource @=@ 0004 @=@ Contributions to Conferences -dnet:publication_resource @=@ 0004 @=@ International Conference -dnet:publication_resource @=@ 0004 @=@ International Conference Abstract/Poster -dnet:publication_resource @=@ 0004 @=@ International Conference ISI/JCR -dnet:publication_resource @=@ 0004 @=@ International Conference communication/abstract/poster -dnet:publication_resource @=@ 0004 @=@ National Conference -dnet:publication_resource @=@ 0004 @=@ National Conference Abstract/Poster -dnet:publication_resource @=@ 0004 @=@ National Conference communication/abstract/poster -dnet:publication_resource @=@ 0004 @=@ PREFACE_PROCEEDINGS -dnet:publication_resource @=@ 0004 @=@ PROCEEDING_PAPER -dnet:publication_resource @=@ 0004 @=@ Papers in Conference Proceedings -dnet:publication_resource @=@ 0004 @=@ Ponencia - Conference or Workshop Item -dnet:publication_resource @=@ 0004 @=@ Presentación -dnet:publication_resource @=@ 0004 @=@ Presentation -dnet:publication_resource @=@ 0004 @=@ Proceedings (peer-reviewed) -dnet:publication_resource @=@ 0004 @=@ Proceedings of a Conference -dnet:publication_resource @=@ 0004 @=@ Proceedings paper -dnet:publication_resource @=@ 0004 @=@ Póster -dnet:publication_resource @=@ 0004 @=@ Póster de congreso -dnet:publication_resource @=@ 0004 @=@ Resumen comunicación Congreso -dnet:publication_resource @=@ 0004 @=@ Resúmen comunicación Congreso -dnet:publication_resource @=@ 0004 @=@ actas de congreso -dnet:publication_resource @=@ 0004 @=@ actes_congres -dnet:publication_resource @=@ 0004 @=@ communication_avec_actes -dnet:publication_resource @=@ 0004 @=@ communication_invitee -dnet:publication_resource @=@ 0004 @=@ communication_par_affiche -dnet:publication_resource @=@ 0004 @=@ communication_sans_actes -dnet:publication_resource @=@ 0004 @=@ conference -dnet:publication_resource @=@ 0004 @=@ conference item -dnet:publication_resource @=@ 0004 @=@ conference proceeding -dnet:publication_resource @=@ 0004 @=@ conferenceObject -dnet:publication_resource @=@ 0004 @=@ conference_paper -dnet:publication_resource @=@ 0004 @=@ doc-type:conferenceObject -dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_18co -dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_18cp -dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_5794 -dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_6670 -dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_c94f -dnet:publication_resource @=@ 0004 @=@ http://purl.org/coar/resource_type/c_f744 -dnet:publication_resource @=@ 0004 @=@ http://purl.org/eprint/type/ConferencePaper -dnet:publication_resource @=@ 0004 @=@ http://purl.org/eprint/type/ConferencePoster -dnet:publication_resource @=@ 0004 @=@ http://purl.org/escidoc/metadata/ves/publication-types/conference-report -dnet:publication_resource @=@ 0004 @=@ http://purl.org/escidoc/metadata/ves/publication-types/proceedings -dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/Conference Paper -dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferenceItem -dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferenceObject -dnet:publication_resource @=@ 0004 @=@ info:eu-repo/semantics/conferencePaper -dnet:publication_resource @=@ 0004 @=@ invited conference talk -dnet:publication_resource @=@ 0004 @=@ plakat -dnet:publication_resource @=@ 0004 @=@ ponencia -dnet:publication_resource @=@ 0004 @=@ poster -dnet:publication_resource @=@ 0004 @=@ prezentacja -dnet:publication_resource @=@ 0004 @=@ proceeding, seminar, workshop without peer review -dnet:publication_resource @=@ 0004 @=@ proceedings -dnet:publication_resource @=@ 0004 @=@ proceedings-article -dnet:publication_resource @=@ 0004 @=@ научный доклад -dnet:publication_resource @=@ 0005 @=@ Magazine Article -dnet:publication_resource @=@ 0005 @=@ Newspaper Article -dnet:publication_resource @=@ 0005 @=@ Newspaper or magazine article -dnet:publication_resource @=@ 0005 @=@ contributionToPeriodical -dnet:publication_resource @=@ 0005 @=@ http://purl.org/coar/resource_type/c_998f -dnet:publication_resource @=@ 0005 @=@ info:eu-repo/semantics/contributionToPeriodical -dnet:publication_resource @=@ 0005 @=@ revista divulgativa -dnet:publication_resource @=@ 0045 @=@ Data Management Plan -dnet:publication_resource @=@ 0045 @=@ Data Management Plan (NSF Generic) -dnet:publication_resource @=@ 0045 @=@ http://purl.org/coar/resource_type/c_ab20 -dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataManagementPolicy -dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataManagementPolicyDocument -dnet:publication_resource @=@ 0045 @=@ http://purl.org/spar/fabio/DataMangementPlan -dnet:publication_resource @=@ 0045 @=@ plan de gestión de datos -dnet:publication_resource @=@ 0045 @=@ publication-datamanagementplan -dnet:publication_resource @=@ 0031 @=@ Data Descriptor -dnet:publication_resource @=@ 0031 @=@ DataPaper -dnet:publication_resource @=@ 0031 @=@ data-article -dnet:publication_resource @=@ 0031 @=@ http://purl.org/coar/resource_type/c_beb9 -dnet:publication_resource @=@ 0021 @=@ Chemical Structures -dnet:publication_resource @=@ 0021 @=@ Dataset/Dataset -dnet:publication_resource @=@ 0021 @=@ Research Data -dnet:publication_resource @=@ 0021 @=@ dataset -dnet:publication_resource @=@ 0021 @=@ http://purl.org/coar/resource_type/c_ddb1 -dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/DDIInstance -dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/datafile -dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/dataset -dnet:publication_resource @=@ 0021 @=@ info:eu-repo/semantics/enhancedObjectFile -dnet:publication_resource @=@ 0006 @=@ Daktaro disertacija / Doctoral dissertation -dnet:publication_resource @=@ 0006 @=@ Diss -dnet:publication_resource @=@ 0006 @=@ Dissertation -dnet:publication_resource @=@ 0006 @=@ Doctoral -dnet:publication_resource @=@ 0006 @=@ DoctoralThesis -dnet:publication_resource @=@ 0006 @=@ PhD Theses -dnet:publication_resource @=@ 0006 @=@ PhD thesis -dnet:publication_resource @=@ 0006 @=@ Tenure-Promotion -dnet:publication_resource @=@ 0006 @=@ Tesi di dottorato -dnet:publication_resource @=@ 0006 @=@ Tesis -dnet:publication_resource @=@ 0006 @=@ Text.Thesis.Doctoral -dnet:publication_resource @=@ 0006 @=@ Theses -dnet:publication_resource @=@ 0006 @=@ Thesis -dnet:publication_resource @=@ 0006 @=@ Thesis or Dissertation -dnet:publication_resource @=@ 0006 @=@ Thesis.Doctoral -dnet:publication_resource @=@ 0006 @=@ doc-type:doctoralThesis -dnet:publication_resource @=@ 0006 @=@ dok -dnet:publication_resource @=@ 0006 @=@ doktori dolgozat -dnet:publication_resource @=@ 0006 @=@ http://purl.org/coar/resource_type/c_db06 -dnet:publication_resource @=@ 0006 @=@ http://purl.org/eprint/type/Thesis -dnet:publication_resource @=@ 0006 @=@ info:eu-repo/semantics/doctoralThesis -dnet:publication_resource @=@ 0006 @=@ tesis doctoral -dnet:publication_resource @=@ 0006 @=@ these -dnet:publication_resource @=@ 0006 @=@ these exercice -dnet:publication_resource @=@ 0023 @=@ Event/Event -dnet:publication_resource @=@ 0023 @=@ event -dnet:publication_resource @=@ 0009 @=@ Departmental Technical Report -dnet:publication_resource @=@ 0009 @=@ Informe Técnico -dnet:publication_resource @=@ 0009 @=@ RESEARCH_REPORT -dnet:publication_resource @=@ 0009 @=@ Tech-Report -dnet:publication_resource @=@ 0009 @=@ Technical Report -dnet:publication_resource @=@ 0009 @=@ http://purl.org/coar/resource_type/c_18gh -dnet:publication_resource @=@ 0009 @=@ informe a organismo financiador -dnet:publication_resource @=@ 0009 @=@ research report -dnet:publication_resource @=@ 0024 @=@ Video -dnet:publication_resource @=@ 0024 @=@ film -dnet:publication_resource @=@ 0024 @=@ http://purl.org/coar/resource_type/c_12ce -dnet:publication_resource @=@ 0024 @=@ http://purl.org/coar/resource_type/c_8a7e -dnet:publication_resource @=@ 0024 @=@ vídeo -dnet:publication_resource @=@ 0025 @=@ Image/Image -dnet:publication_resource @=@ 0025 @=@ Imagen -dnet:publication_resource @=@ 0025 @=@ Imagen - Image -dnet:publication_resource @=@ 0025 @=@ Imagen 3-D -dnet:publication_resource @=@ 0025 @=@ fotó -dnet:publication_resource @=@ 0025 @=@ grafika -dnet:publication_resource @=@ 0025 @=@ http://purl.org/coar/resource_type/c_ecc8 -dnet:publication_resource @=@ 0025 @=@ image -dnet:publication_resource @=@ 0025 @=@ image-diagram -dnet:publication_resource @=@ 0025 @=@ image-drawing -dnet:publication_resource @=@ 0025 @=@ image-figure -dnet:publication_resource @=@ 0025 @=@ image-other -dnet:publication_resource @=@ 0025 @=@ image-photo -dnet:publication_resource @=@ 0025 @=@ image-plot -dnet:publication_resource @=@ 0025 @=@ Drawing -dnet:publication_resource @=@ 0025 @=@ Photo -dnet:publication_resource @=@ 0025 @=@ Figure -dnet:publication_resource @=@ 0025 @=@ Diagram -dnet:publication_resource @=@ 0025 @=@ Plot -dnet:publication_resource @=@ 0026 @=@ http://purl.org/coar/resource_type/c_e9a0 -dnet:publication_resource @=@ 0026 @=@ interactiveResource -dnet:publication_resource @=@ 0011 @=@ Internal note -dnet:publication_resource @=@ 0011 @=@ http://purl.org/coar/resource_type/c_18ww -dnet:publication_resource @=@ 0043 @=@ http://purl.org/coar/resource_type/c_0640 -dnet:publication_resource @=@ 0010 @=@ Inaugural lecture -dnet:publication_resource @=@ 0010 @=@ Material didáctico -dnet:publication_resource @=@ 0010 @=@ Public-Lecture -dnet:publication_resource @=@ 0010 @=@ Teaching Resource -dnet:publication_resource @=@ 0010 @=@ http://purl.org/coar/resource_type/c_8544 -dnet:publication_resource @=@ 0010 @=@ info:eu-repo/semantics/lecture -dnet:publication_resource @=@ 0010 @=@ lesson -dnet:publication_resource @=@ 0010 @=@ Учебный материал -dnet:publication_resource @=@ 0007 @=@ Diploma Project -dnet:publication_resource @=@ 0007 @=@ MSc Thesis -dnet:publication_resource @=@ 0007 @=@ Magistro darbas / Master thesis -dnet:publication_resource @=@ 0007 @=@ Master Degree -dnet:publication_resource @=@ 0007 @=@ Master's -dnet:publication_resource @=@ 0007 @=@ Master's Degree -dnet:publication_resource @=@ 0007 @=@ Masterarbeit u.a. -dnet:publication_resource @=@ 0007 @=@ Masters (Taught) -dnet:publication_resource @=@ 0007 @=@ Masters thesis -dnet:publication_resource @=@ 0007 @=@ Masters-Thesis.Magister -dnet:publication_resource @=@ 0007 @=@ Tesina -dnet:publication_resource @=@ 0007 @=@ Thesis.Master -dnet:publication_resource @=@ 0007 @=@ Trabajo fin de Máster -dnet:publication_resource @=@ 0007 @=@ doc-type:masterThesis -dnet:publication_resource @=@ 0007 @=@ hdr -dnet:publication_resource @=@ 0007 @=@ http://purl.org/coar/resource_type/c_bdcc -dnet:publication_resource @=@ 0007 @=@ info:eu-repo/semantics/masterThesis -dnet:publication_resource @=@ 0007 @=@ masterThesis -dnet:publication_resource @=@ 0007 @=@ memoire -dnet:publication_resource @=@ 0007 @=@ tesis de maestría -dnet:publication_resource @=@ 0027 @=@ Model/Model -dnet:publication_resource @=@ 0027 @=@ model -dnet:publication_resource @=@ 0012 @=@ Newsletter Article -dnet:publication_resource @=@ 0020 @=@ Estudio y edición crítica de música -dnet:publication_resource @=@ 0020 @=@ Exhibition -dnet:publication_resource @=@ 0020 @=@ Kita / Other -dnet:publication_resource @=@ 0020 @=@ Learning Object -dnet:publication_resource @=@ 0020 @=@ Mapa -dnet:publication_resource @=@ 0020 @=@ Modelo de utilidad -dnet:publication_resource @=@ 0020 @=@ Online Resource -dnet:publication_resource @=@ 0020 @=@ Otro - Other -dnet:publication_resource @=@ 0020 @=@ PEDAGOGICAL_DOCUMENT -dnet:publication_resource @=@ 0020 @=@ Partitura -dnet:publication_resource @=@ 0020 @=@ Planimetría -dnet:publication_resource @=@ 0020 @=@ Registered Copyright -dnet:publication_resource @=@ 0020 @=@ Research Tool -dnet:publication_resource @=@ 0020 @=@ Sitio web -dnet:publication_resource @=@ 0020 @=@ Trabajo de divulgación -dnet:publication_resource @=@ 0020 @=@ University Academic Unit -dnet:publication_resource @=@ 0020 @=@ Web publication/site -dnet:publication_resource @=@ 0020 @=@ application -dnet:publication_resource @=@ 0020 @=@ artefact -dnet:publication_resource @=@ 0020 @=@ carte -dnet:publication_resource @=@ 0020 @=@ composition -dnet:publication_resource @=@ 0020 @=@ document_audiovisuel -dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_12cc -dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_12cd -dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_1843 -dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_18cd -dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_18cw -dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_26e4 -dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_7ad9 -dnet:publication_resource @=@ 0020 @=@ http://purl.org/coar/resource_type/c_e059 -dnet:publication_resource @=@ 0020 @=@ info:eu-repo/semantics/other -dnet:publication_resource @=@ 0020 @=@ learningObject -dnet:publication_resource @=@ 0020 @=@ map -dnet:publication_resource @=@ 0020 @=@ misc -dnet:publication_resource @=@ 0020 @=@ other -dnet:publication_resource @=@ 0020 @=@ otro -dnet:publication_resource @=@ 0020 @=@ revue -dnet:publication_resource @=@ 0038 @=@ Abstract -dnet:publication_resource @=@ 0038 @=@ Blog -dnet:publication_resource @=@ 0038 @=@ Book Prospectus -dnet:publication_resource @=@ 0038 @=@ Dictionary Entry -dnet:publication_resource @=@ 0038 @=@ Disclosure -dnet:publication_resource @=@ 0038 @=@ Editorial -dnet:publication_resource @=@ 0038 @=@ Editorial ISI/JCR -dnet:publication_resource @=@ 0038 @=@ Editors -dnet:publication_resource @=@ 0038 @=@ Editors (non peer-reviewed) -dnet:publication_resource @=@ 0038 @=@ Encyclopedia Entry -dnet:publication_resource @=@ 0038 @=@ Entrada de blog -dnet:publication_resource @=@ 0038 @=@ Funding Submission -dnet:publication_resource @=@ 0038 @=@ HabilitationThesis -dnet:publication_resource @=@ 0038 @=@ License -dnet:publication_resource @=@ 0038 @=@ Manual -dnet:publication_resource @=@ 0038 @=@ Manuscript -dnet:publication_resource @=@ 0038 @=@ Manuscrito -dnet:publication_resource @=@ 0038 @=@ Other publication (non peer-review) -dnet:publication_resource @=@ 0038 @=@ Other publication (peer-review) -dnet:publication_resource @=@ 0038 @=@ Revista -dnet:publication_resource @=@ 0038 @=@ Supervised Student Publication -dnet:publication_resource @=@ 0038 @=@ Tesis/trabajos de grado – Thesis -dnet:publication_resource @=@ 0038 @=@ Text -dnet:publication_resource @=@ 0038 @=@ Text/Text -dnet:publication_resource @=@ 0038 @=@ Trademark -dnet:publication_resource @=@ 0038 @=@ afterword -dnet:publication_resource @=@ 0038 @=@ avantpropos -dnet:publication_resource @=@ 0038 @=@ bibliography -dnet:publication_resource @=@ 0038 @=@ chronique -dnet:publication_resource @=@ 0038 @=@ compte rendu -dnet:publication_resource @=@ 0038 @=@ correction -dnet:publication_resource @=@ 0038 @=@ corrigenda -dnet:publication_resource @=@ 0038 @=@ foreword -dnet:publication_resource @=@ 0038 @=@ habilitation à diriger des recherches -dnet:publication_resource @=@ 0038 @=@ historicalDocument -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_0040 -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_0857 -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_18cf -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_18wz -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_3e5a -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_46ec -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_6947 -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_7acd -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_86bc -dnet:publication_resource @=@ 0038 @=@ http://purl.org/coar/resource_type/c_b239 -dnet:publication_resource @=@ 0038 @=@ note de lecture -dnet:publication_resource @=@ 0038 @=@ notedelecture -dnet:publication_resource @=@ 0038 @=@ other publication -dnet:publication_resource @=@ 0038 @=@ postface -dnet:publication_resource @=@ 0038 @=@ publication-other -dnet:publication_resource @=@ 0038 @=@ revuedepresse -dnet:publication_resource @=@ 0038 @=@ sa_component -dnet:publication_resource @=@ 0038 @=@ standard -dnet:publication_resource @=@ 0038 @=@ standard-series -dnet:publication_resource @=@ 0038 @=@ Índice -dnet:publication_resource @=@ 0013 @=@ A3 Kirjan tai muun kokoomateoksen osa -dnet:publication_resource @=@ 0013 @=@ Book Part (author) -dnet:publication_resource @=@ 0013 @=@ Book Section / Chapter -dnet:publication_resource @=@ 0013 @=@ Book chapter or Essay in book -dnet:publication_resource @=@ 0013 @=@ Book editorial -dnet:publication_resource @=@ 0013 @=@ Book section -dnet:publication_resource @=@ 0013 @=@ Book_Chapter -dnet:publication_resource @=@ 0013 @=@ Buchbeitrag -dnet:publication_resource @=@ 0013 @=@ Capítulo de libro -dnet:publication_resource @=@ 0013 @=@ Contribution to International Book/Monograph -dnet:publication_resource @=@ 0013 @=@ Contribution to International Book/Monograph ISI/JCR -dnet:publication_resource @=@ 0013 @=@ Contribution to National Book/Monograph -dnet:publication_resource @=@ 0013 @=@ Contribution to book (non peer-reviewed) -dnet:publication_resource @=@ 0013 @=@ Contribution to book (peer-reviewed) -dnet:publication_resource @=@ 0013 @=@ Knygos dalis / Book chapter -dnet:publication_resource @=@ 0013 @=@ Part of book - chapter -dnet:publication_resource @=@ 0013 @=@ Rozdział z książki -dnet:publication_resource @=@ 0013 @=@ book chapter -dnet:publication_resource @=@ 0013 @=@ book-chapter -dnet:publication_resource @=@ 0013 @=@ book-part -dnet:publication_resource @=@ 0013 @=@ book-section -dnet:publication_resource @=@ 0013 @=@ bookPart -dnet:publication_resource @=@ 0013 @=@ book_content -dnet:publication_resource @=@ 0013 @=@ chapitre_ouvrage -dnet:publication_resource @=@ 0013 @=@ chapter -dnet:publication_resource @=@ 0013 @=@ doc-type:bookPart -dnet:publication_resource @=@ 0013 @=@ http://purl.org/coar/resource_type/c_3248 -dnet:publication_resource @=@ 0013 @=@ http://purl.org/eprint/type/BookItem -dnet:publication_resource @=@ 0013 @=@ info:eu-repo/semantics/Chapter -dnet:publication_resource @=@ 0013 @=@ info:eu-repo/semantics/bookPart -dnet:publication_resource @=@ 0013 @=@ könyvfejezet -dnet:publication_resource @=@ 0013 @=@ publication-section -dnet:publication_resource @=@ 0013 @=@ reference-entry -dnet:publication_resource @=@ 0013 @=@ reference_entry -dnet:publication_resource @=@ 0013 @=@ scientific book chapter -dnet:publication_resource @=@ 0013 @=@ Глава монографии -dnet:publication_resource @=@ 0019 @=@ H1 Myönnetty patentti -dnet:publication_resource @=@ 0019 @=@ Patente -dnet:publication_resource @=@ 0019 @=@ Solicitud de patente -dnet:publication_resource @=@ 0019 @=@ Traducción de patente -dnet:publication_resource @=@ 0019 @=@ brevet -dnet:publication_resource @=@ 0019 @=@ http://purl.org/coar/resource_type/c_15cd -dnet:publication_resource @=@ 0019 @=@ info:eu-repo/semantics/patent -dnet:publication_resource @=@ 0028 @=@ Service -dnet:publication_resource @=@ 0028 @=@ physicalObject -dnet:publication_resource @=@ 0016 @=@ Pre Print -dnet:publication_resource @=@ 0016 @=@ Pre-print -dnet:publication_resource @=@ 0016 @=@ http://purl.org/coar/resource_type/c_816b -dnet:publication_resource @=@ 0016 @=@ info:eu-repo/semantics/preprint -dnet:publication_resource @=@ 0016 @=@ Препринт -dnet:publication_resource @=@ 0034 @=@ http://purl.org/coar/resource_type/c_18op -dnet:publication_resource @=@ 0034 @=@ publication-deliverable -dnet:publication_resource @=@ 0034 @=@ Project deliverable -dnet:publication_resource @=@ 0035 @=@ publication-milestone -dnet:publication_resource @=@ 0035 @=@ Project milestone -dnet:publication_resource @=@ 0036 @=@ http://purl.org/coar/resource_type/c_baaf -dnet:publication_resource @=@ 0036 @=@ research-proposal -dnet:publication_resource @=@ 0036 @=@ Proposal -dnet:publication_resource @=@ 0017 @=@ ACTIVITY_REPORT -dnet:publication_resource @=@ 0017 @=@ Case Report -dnet:publication_resource @=@ 0017 @=@ Commissioned report -dnet:publication_resource @=@ 0017 @=@ D4 Julkaistu kehittämis- tai tutkimusraportti tai -selvitys -dnet:publication_resource @=@ 0017 @=@ Deliverable -dnet:publication_resource @=@ 0017 @=@ Documento tecnico -dnet:publication_resource @=@ 0017 @=@ EUR - Scientific and Technical Research Reports -dnet:publication_resource @=@ 0017 @=@ JRC Reference Reports -dnet:publication_resource @=@ 0017 @=@ Project Report -dnet:publication_resource @=@ 0017 @=@ brief-report -dnet:publication_resource @=@ 0017 @=@ case-report -dnet:publication_resource @=@ 0017 @=@ chapitre_rapport -dnet:publication_resource @=@ 0017 @=@ doc-type:report -dnet:publication_resource @=@ 0017 @=@ document_institutionnel -dnet:publication_resource @=@ 0017 @=@ document_technique -dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_186u -dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18hj -dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18wq -dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_18ws -dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_71bd -dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_93fc -dnet:publication_resource @=@ 0017 @=@ http://purl.org/coar/resource_type/c_ba1f -dnet:publication_resource @=@ 0017 @=@ http://purl.org/eprint/type/Report -dnet:publication_resource @=@ 0017 @=@ info:eu-repo/semantics/report -dnet:publication_resource @=@ 0017 @=@ publication-softwaredocumentation -dnet:publication_resource @=@ 0017 @=@ rapport_expertise -dnet:publication_resource @=@ 0017 @=@ rapport_mission -dnet:publication_resource @=@ 0017 @=@ report -dnet:publication_resource @=@ 0017 @=@ report-paper -dnet:publication_resource @=@ 0017 @=@ report-paper_title -dnet:publication_resource @=@ 0017 @=@ report-series -dnet:publication_resource @=@ 0017 @=@ support_cours -dnet:publication_resource @=@ 0017 @=@ Software documentation -dnet:publication_resource @=@ 0014 @=@ Arbeitspapier -dnet:publication_resource @=@ 0014 @=@ Departmental Bulletin Paper -dnet:publication_resource @=@ 0014 @=@ Documento de trabajo -dnet:publication_resource @=@ 0014 @=@ Paper -dnet:publication_resource @=@ 0014 @=@ Project description -dnet:publication_resource @=@ 0014 @=@ Research-Paper -dnet:publication_resource @=@ 0014 @=@ ResearchPaper -dnet:publication_resource @=@ 0014 @=@ Working / discussion paper -dnet:publication_resource @=@ 0014 @=@ Working Paper -dnet:publication_resource @=@ 0014 @=@ Working Paper / Technical Report -dnet:publication_resource @=@ 0014 @=@ doc-type:workingPaper -dnet:publication_resource @=@ 0014 @=@ http://purl.org/coar/resource_type/c_8042 -dnet:publication_resource @=@ 0014 @=@ http://purl.org/escidoc/metadata/ves/publication-types/paper -dnet:publication_resource @=@ 0014 @=@ info:eu-repo/semantics/paper -dnet:publication_resource @=@ 0014 @=@ info:eu-repo/semantics/workingPaper -dnet:publication_resource @=@ 0014 @=@ workingPaper -dnet:publication_resource @=@ 0015 @=@ A2 Katsausartikkeli tieteellisessä aikakauslehdessä -dnet:publication_resource @=@ 0015 @=@ Book Review -dnet:publication_resource @=@ 0015 @=@ Book/Film/Article review -dnet:publication_resource @=@ 0015 @=@ Literature review -dnet:publication_resource @=@ 0015 @=@ Peer review -dnet:publication_resource @=@ 0015 @=@ Reseña bibliográfica -dnet:publication_resource @=@ 0015 @=@ Review Article -dnet:publication_resource @=@ 0015 @=@ Revisión -dnet:publication_resource @=@ 0015 @=@ RezensionReview -dnet:publication_resource @=@ 0015 @=@ book-review -dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_ba08 -dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_dcae04bc -dnet:publication_resource @=@ 0015 @=@ http://purl.org/coar/resource_type/c_efa0 -dnet:publication_resource @=@ 0015 @=@ info:eu-repo/semantics/review -dnet:publication_resource @=@ 0015 @=@ peer-review -dnet:publication_resource @=@ 0015 @=@ reseña de libro -dnet:publication_resource @=@ 0029 @=@ Software -dnet:publication_resource @=@ 0029 @=@ Software/Software -dnet:publication_resource @=@ 0029 @=@ Workflow -dnet:publication_resource @=@ 0029 @=@ Workflow/Workflow -dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_393c -dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_5ce6 -dnet:publication_resource @=@ 0029 @=@ http://purl.org/coar/resource_type/c_c950 -dnet:publication_resource @=@ 0032 @=@ http://purl.org/coar/resource_type/c_7bab -dnet:publication_resource @=@ 0030 @=@ Audio -dnet:publication_resource @=@ 0030 @=@ http://purl.org/coar/resource_type/c_18cc -dnet:publication_resource @=@ 0030 @=@ sound -dnet:publication_resource @=@ 0044 @=@ Graduate diploma -dnet:publication_resource @=@ 0044 @=@ Undergraduate diploma +dnet:pid_types @=@ orcid @=@ ORCID12 +dnet:review_levels @=@ 0000 @=@ UNKNOWN dnet:review_levels @=@ 0002 @=@ 80 大阪経大学会「Working Paper」 dnet:review_levels @=@ 0002 @=@ AO dnet:review_levels @=@ 0002 @=@ ARTICLE SANS COMITE DE LECTURE (ASCL) @@ -1025,6 +977,7 @@ dnet:review_levels @=@ 0002 @=@ Discussion Paper dnet:review_levels @=@ 0002 @=@ Document de travail (Working Paper) dnet:review_levels @=@ 0002 @=@ Documento de trabajo dnet:review_levels @=@ 0002 @=@ Documento de trabajo de investigaci??n +dnet:review_levels @=@ 0002 @=@ Draft dnet:review_levels @=@ 0002 @=@ E-pub ahead of print dnet:review_levels @=@ 0002 @=@ Editorial de revista, no evaluado por pares dnet:review_levels @=@ 0002 @=@ Editorial de revista, não avaliado por pares @@ -1095,7 +1048,6 @@ dnet:review_levels @=@ 0002 @=@ communication_invitee dnet:review_levels @=@ 0002 @=@ doc-type:preprint dnet:review_levels @=@ 0002 @=@ doc-type:workingPaper dnet:review_levels @=@ 0002 @=@ draf -dnet:review_levels @=@ 0002 @=@ draft dnet:review_levels @=@ 0002 @=@ eu-repo/semantics/submittedVersion dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/resource_type/c_8042 dnet:review_levels @=@ 0002 @=@ http://purl.org/coar/resource_type/c_816b @@ -1178,6 +1130,7 @@ dnet:review_levels @=@ 0001 @=@ Artículo de investigación, Investigaciones ori dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares, Ensayos de investigación dnet:review_levels @=@ 0001 @=@ Artículo evaluado por pares, Investigaciones empíricas, Artículos de investigación +dnet:review_levels @=@ 0001 @=@ Artículo revisado dnet:review_levels @=@ 0001 @=@ Artículo revisado por pares dnet:review_levels @=@ 0001 @=@ Artículos de estudiantes, Artículo evaluado por pares, Artículos de investigación dnet:review_levels @=@ 0001 @=@ Artículos de investigación evaluados por doble ciego @@ -1185,7 +1138,7 @@ dnet:review_levels @=@ 0001 @=@ Artículos evaluadores por doble ciego dnet:review_levels @=@ 0001 @=@ Artículos evaluados por pares dnet:review_levels @=@ 0001 @=@ Artículos evaluados por pares académicos dnet:review_levels @=@ 0001 @=@ Artículos revisados por pares -dnet:review_levels @=@ 0001 @=@ avaliadas pelos pares +dnet:review_levels @=@ 0001 @=@ Avaliadas pelos pares dnet:review_levels @=@ 0001 @=@ Avaliado anonimamente por pares dnet:review_levels @=@ 0001 @=@ Avaliado em duplo cego por pares dnet:review_levels @=@ 0001 @=@ Avaliado pela Editoria @@ -1236,6 +1189,7 @@ dnet:review_levels @=@ 0001 @=@ Peer-reviewed Review dnet:review_levels @=@ 0001 @=@ Peer-reviewed Review Article dnet:review_levels @=@ 0001 @=@ Peer-reviewed Text dnet:review_levels @=@ 0001 @=@ Peer-reviewed communication +dnet:review_levels @=@ 0001 @=@ Peer-reviewed conference proceedings dnet:review_levels @=@ 0001 @=@ Peer-reviewed research article dnet:review_levels @=@ 0001 @=@ Peer-reviewed short communication dnet:review_levels @=@ 0001 @=@ PeerReviewed diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt index 1daca3ca9..59bed7c3a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt @@ -1,414 +1,39 @@ ModularUiLabels @=@ ModularUiLabels @=@ PendingRepositoryResources @=@ Pending datasource ModularUiLabels @=@ ModularUiLabels @=@ RepositoryServiceResources @=@ Valid datasource -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/OPENACCESS_VERSION @=@ Another Open Access version of a publication -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/PID @=@ Another persistent identifier associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/MESHEUROPMC @=@ Another classification term from the Medical Subject Headings (MeSH) that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/DDC @=@ Another Dewey Decimal classification term (DDC) that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/ACM @=@ Another ACM classification term that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/JEL @=@ Another Journal of Economic Literature (JEL) classification term that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/ARXIV @=@ Another ARXIV classification term that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PID @=@ A persistent identifier associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PROJECT @=@ A project reference that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/ABSTRACT @=@ An abstract describing among your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/MESHEUROPMC @=@ A classification term from the Medical Subject Headings (MeSH) that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/DDC @=@ A Dewey Decimal classification term (DDC) that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/ACM @=@ An ACM classification term that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/JEL @=@ A Journal of Economic Literature (JEL) classification term that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/ARXIV @=@ An ARXIV classification term that can be associated to your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION_DATE @=@ A date of publication missing in your content -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/OPENACCESS_VERSION @=@ An Open Access versions of your publications -dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/AUTHOR/ORCID @=@ An Open Researcher and Contributor ID (ORCID) that can be associated to an author of your publications -datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsCitedBy @=@ IsCitedBy -datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsNewVersionOf @=@ IsNewVersionOf -datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsPartOf @=@ IsPartOf -datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsPreviousVersionOf @=@ IsPreviousVersionOf -datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsReferencedBy @=@ IsReferencedBy -datacite:relation_typologies @=@ datacite:relation_typologies @=@ References @=@ References -datacite:relation_typologies @=@ datacite:relation_typologies @=@ UNKNOWN @=@ UNKNOWN -datacite:id_typologies @=@ datacite:id_typologies @=@ ARK @=@ ARK -datacite:id_typologies @=@ datacite:id_typologies @=@ DOI @=@ DOI -datacite:id_typologies @=@ datacite:id_typologies @=@ EAN13 @=@ EAN13 -datacite:id_typologies @=@ datacite:id_typologies @=@ EISSN @=@ EISSN -datacite:id_typologies @=@ datacite:id_typologies @=@ Handle @=@ Handle -datacite:id_typologies @=@ datacite:id_typologies @=@ ISBN @=@ ISBN -datacite:id_typologies @=@ datacite:id_typologies @=@ ISSN @=@ ISSN -datacite:id_typologies @=@ datacite:id_typologies @=@ ISTC @=@ ISTC -datacite:id_typologies @=@ datacite:id_typologies @=@ LISSN @=@ LISSN -datacite:id_typologies @=@ datacite:id_typologies @=@ LSID @=@ LSID -datacite:id_typologies @=@ datacite:id_typologies @=@ PURL @=@ PURL -datacite:id_typologies @=@ datacite:id_typologies @=@ UNKNOWN @=@ UNKNOWN -datacite:id_typologies @=@ datacite:id_typologies @=@ UPC @=@ UPC -datacite:id_typologies @=@ datacite:id_typologies @=@ URL @=@ URL -datacite:id_typologies @=@ datacite:id_typologies @=@ URN @=@ URN -dnet:countries @=@ dnet:countries @=@ AF @=@ Afghanistan -dnet:countries @=@ dnet:countries @=@ AL @=@ Albania -dnet:countries @=@ dnet:countries @=@ DZ @=@ Algeria -dnet:countries @=@ dnet:countries @=@ AS @=@ American Samoa -dnet:countries @=@ dnet:countries @=@ AD @=@ Andorra -dnet:countries @=@ dnet:countries @=@ AO @=@ Angola -dnet:countries @=@ dnet:countries @=@ AI @=@ Anguilla -dnet:countries @=@ dnet:countries @=@ AQ @=@ Antarctica -dnet:countries @=@ dnet:countries @=@ AG @=@ Antigua and Barbuda -dnet:countries @=@ dnet:countries @=@ AR @=@ Argentina -dnet:countries @=@ dnet:countries @=@ AM @=@ Armenia -dnet:countries @=@ dnet:countries @=@ AW @=@ Aruba -dnet:countries @=@ dnet:countries @=@ AU @=@ Australia -dnet:countries @=@ dnet:countries @=@ AT @=@ Austria -dnet:countries @=@ dnet:countries @=@ AZ @=@ Azerbaijan -dnet:countries @=@ dnet:countries @=@ BS @=@ Bahamas -dnet:countries @=@ dnet:countries @=@ BH @=@ Bahrain -dnet:countries @=@ dnet:countries @=@ BD @=@ Bangladesh -dnet:countries @=@ dnet:countries @=@ BB @=@ Barbados -dnet:countries @=@ dnet:countries @=@ BY @=@ Belarus -dnet:countries @=@ dnet:countries @=@ BE @=@ Belgium -dnet:countries @=@ dnet:countries @=@ BZ @=@ Belize -dnet:countries @=@ dnet:countries @=@ BJ @=@ Benin -dnet:countries @=@ dnet:countries @=@ BM @=@ Bermuda -dnet:countries @=@ dnet:countries @=@ BT @=@ Bhutan -dnet:countries @=@ dnet:countries @=@ BO @=@ Bolivia -dnet:countries @=@ dnet:countries @=@ BQ @=@ Bonaire, Sint Eustatius and Saba -dnet:countries @=@ dnet:countries @=@ BA @=@ Bosnia and Herzegovina -dnet:countries @=@ dnet:countries @=@ BW @=@ Botswana -dnet:countries @=@ dnet:countries @=@ BV @=@ Bouvet Island -dnet:countries @=@ dnet:countries @=@ BR @=@ Brazil -dnet:countries @=@ dnet:countries @=@ IO @=@ British Indian Ocean Territory -dnet:countries @=@ dnet:countries @=@ BN @=@ Brunei Darussalam -dnet:countries @=@ dnet:countries @=@ BG @=@ Bulgaria -dnet:countries @=@ dnet:countries @=@ BF @=@ Burkina Faso -dnet:countries @=@ dnet:countries @=@ BI @=@ Burundi -dnet:countries @=@ dnet:countries @=@ KH @=@ Cambodia -dnet:countries @=@ dnet:countries @=@ CM @=@ Cameroon -dnet:countries @=@ dnet:countries @=@ CA @=@ Canada -dnet:countries @=@ dnet:countries @=@ CV @=@ Cape Verde -dnet:countries @=@ dnet:countries @=@ KY @=@ Cayman Islands -dnet:countries @=@ dnet:countries @=@ CF @=@ Central African Republic -dnet:countries @=@ dnet:countries @=@ TD @=@ Chad -dnet:countries @=@ dnet:countries @=@ CL @=@ Chile -dnet:countries @=@ dnet:countries @=@ CN @=@ China (People's Republic of) -dnet:countries @=@ dnet:countries @=@ CX @=@ Christmas Island -dnet:countries @=@ dnet:countries @=@ CC @=@ Cocos (Keeling) Islands -dnet:countries @=@ dnet:countries @=@ CO @=@ Colombia -dnet:countries @=@ dnet:countries @=@ KM @=@ Comoros -dnet:countries @=@ dnet:countries @=@ CG @=@ Congo -dnet:countries @=@ dnet:countries @=@ CD @=@ Congo (Democratic Republic of) -dnet:countries @=@ dnet:countries @=@ CK @=@ Cook Islands -dnet:countries @=@ dnet:countries @=@ CR @=@ Costa Rica -dnet:countries @=@ dnet:countries @=@ CI @=@ Cote d'Ivoire -dnet:countries @=@ dnet:countries @=@ HR @=@ Croatia -dnet:countries @=@ dnet:countries @=@ CU @=@ Cuba -dnet:countries @=@ dnet:countries @=@ CW @=@ Curaçao -dnet:countries @=@ dnet:countries @=@ CY @=@ Cyprus -dnet:countries @=@ dnet:countries @=@ CZ @=@ Czech Republic -dnet:countries @=@ dnet:countries @=@ DK @=@ Denmark -dnet:countries @=@ dnet:countries @=@ DJ @=@ Djibouti -dnet:countries @=@ dnet:countries @=@ DM @=@ Dominica -dnet:countries @=@ dnet:countries @=@ DO @=@ Dominican Republic -dnet:countries @=@ dnet:countries @=@ EC @=@ Ecuador -dnet:countries @=@ dnet:countries @=@ EG @=@ Egypt -dnet:countries @=@ dnet:countries @=@ SV @=@ El Salvador -dnet:countries @=@ dnet:countries @=@ EN @=@ England -dnet:countries @=@ dnet:countries @=@ GQ @=@ Equatorial Guinea -dnet:countries @=@ dnet:countries @=@ ER @=@ Eritrea -dnet:countries @=@ dnet:countries @=@ EE @=@ Estonia -dnet:countries @=@ dnet:countries @=@ ET @=@ Ethiopia -dnet:countries @=@ dnet:countries @=@ EU @=@ European Union -dnet:countries @=@ dnet:countries @=@ FK @=@ Falkland Islands (Malvinas) -dnet:countries @=@ dnet:countries @=@ FO @=@ Faroe Islands -dnet:countries @=@ dnet:countries @=@ FJ @=@ Fiji -dnet:countries @=@ dnet:countries @=@ FI @=@ Finland -dnet:countries @=@ dnet:countries @=@ MK @=@ Former Yugoslav Republic of Macedonia -dnet:countries @=@ dnet:countries @=@ FR @=@ France -dnet:countries @=@ dnet:countries @=@ GF @=@ French Guiana -dnet:countries @=@ dnet:countries @=@ PF @=@ French Polynesia -dnet:countries @=@ dnet:countries @=@ TF @=@ French Southern Territories -dnet:countries @=@ dnet:countries @=@ GA @=@ Gabon -dnet:countries @=@ dnet:countries @=@ GM @=@ Gambia -dnet:countries @=@ dnet:countries @=@ GE @=@ Georgia -dnet:countries @=@ dnet:countries @=@ DE @=@ Germany -dnet:countries @=@ dnet:countries @=@ GH @=@ Ghana -dnet:countries @=@ dnet:countries @=@ GI @=@ Gibraltar -dnet:countries @=@ dnet:countries @=@ GR @=@ Greece -dnet:countries @=@ dnet:countries @=@ GL @=@ Greenland -dnet:countries @=@ dnet:countries @=@ GD @=@ Grenada -dnet:countries @=@ dnet:countries @=@ GP @=@ Guadeloupe -dnet:countries @=@ dnet:countries @=@ GU @=@ Guam -dnet:countries @=@ dnet:countries @=@ GT @=@ Guatemala -dnet:countries @=@ dnet:countries @=@ GG @=@ Guernsey -dnet:countries @=@ dnet:countries @=@ GN @=@ Guinea -dnet:countries @=@ dnet:countries @=@ GW @=@ Guinea-Bissau -dnet:countries @=@ dnet:countries @=@ GY @=@ Guyana -dnet:countries @=@ dnet:countries @=@ HT @=@ Haiti -dnet:countries @=@ dnet:countries @=@ HM @=@ Heard Island and McDonald Islands -dnet:countries @=@ dnet:countries @=@ VA @=@ Holy See (Vatican City State) -dnet:countries @=@ dnet:countries @=@ HN @=@ Honduras -dnet:countries @=@ dnet:countries @=@ HK @=@ Hong Kong -dnet:countries @=@ dnet:countries @=@ HU @=@ Hungary -dnet:countries @=@ dnet:countries @=@ IS @=@ Iceland -dnet:countries @=@ dnet:countries @=@ IN @=@ India -dnet:countries @=@ dnet:countries @=@ ID @=@ Indonesia -dnet:countries @=@ dnet:countries @=@ IR @=@ Iran (Islamic Republic of) -dnet:countries @=@ dnet:countries @=@ IQ @=@ Iraq -dnet:countries @=@ dnet:countries @=@ IE @=@ Ireland -dnet:countries @=@ dnet:countries @=@ IM @=@ Isle of Man -dnet:countries @=@ dnet:countries @=@ IL @=@ Israel -dnet:countries @=@ dnet:countries @=@ IT @=@ Italy -dnet:countries @=@ dnet:countries @=@ JM @=@ Jamaica -dnet:countries @=@ dnet:countries @=@ JP @=@ Japan -dnet:countries @=@ dnet:countries @=@ JE @=@ Jersey -dnet:countries @=@ dnet:countries @=@ JO @=@ Jordan -dnet:countries @=@ dnet:countries @=@ KZ @=@ Kazakhstan -dnet:countries @=@ dnet:countries @=@ KE @=@ Kenya -dnet:countries @=@ dnet:countries @=@ KI @=@ Kiribati -dnet:countries @=@ dnet:countries @=@ KR @=@ Korea (Republic of) -dnet:countries @=@ dnet:countries @=@ KP @=@ Korea, Democatric People's Republic of -dnet:countries @=@ dnet:countries @=@ XK @=@ Kosovo * UN resolution -dnet:countries @=@ dnet:countries @=@ KW @=@ Kuwait -dnet:countries @=@ dnet:countries @=@ KG @=@ Kyrgyzstan -dnet:countries @=@ dnet:countries @=@ LA @=@ Lao (People's Democratic Republic) -dnet:countries @=@ dnet:countries @=@ LV @=@ Latvia -dnet:countries @=@ dnet:countries @=@ LB @=@ Lebanon -dnet:countries @=@ dnet:countries @=@ LS @=@ Lesotho -dnet:countries @=@ dnet:countries @=@ LR @=@ Liberia -dnet:countries @=@ dnet:countries @=@ LY @=@ Libyan Arab Jamahiriya -dnet:countries @=@ dnet:countries @=@ LI @=@ Liechtenstein -dnet:countries @=@ dnet:countries @=@ LT @=@ Lithuania -dnet:countries @=@ dnet:countries @=@ LU @=@ Luxembourg -dnet:countries @=@ dnet:countries @=@ MO @=@ Macao -dnet:countries @=@ dnet:countries @=@ MG @=@ Madagascar -dnet:countries @=@ dnet:countries @=@ MW @=@ Malawi -dnet:countries @=@ dnet:countries @=@ MY @=@ Malaysia -dnet:countries @=@ dnet:countries @=@ MV @=@ Maldives -dnet:countries @=@ dnet:countries @=@ ML @=@ Mali -dnet:countries @=@ dnet:countries @=@ MT @=@ Malta -dnet:countries @=@ dnet:countries @=@ MH @=@ Marshall Islands -dnet:countries @=@ dnet:countries @=@ MQ @=@ Martinique -dnet:countries @=@ dnet:countries @=@ MR @=@ Mauritania -dnet:countries @=@ dnet:countries @=@ MU @=@ Mauritius -dnet:countries @=@ dnet:countries @=@ YT @=@ Mayotte -dnet:countries @=@ dnet:countries @=@ MX @=@ Mexico -dnet:countries @=@ dnet:countries @=@ FM @=@ Micronesia, Federated States of -dnet:countries @=@ dnet:countries @=@ MD @=@ Moldova (Republic of) -dnet:countries @=@ dnet:countries @=@ MN @=@ Mongolia -dnet:countries @=@ dnet:countries @=@ ME @=@ Montenegro -dnet:countries @=@ dnet:countries @=@ MS @=@ Montserrat -dnet:countries @=@ dnet:countries @=@ MA @=@ Morocco -dnet:countries @=@ dnet:countries @=@ MZ @=@ Mozambique -dnet:countries @=@ dnet:countries @=@ MM @=@ Myanmar -dnet:countries @=@ dnet:countries @=@ NA @=@ Namibia -dnet:countries @=@ dnet:countries @=@ NR @=@ Nauru -dnet:countries @=@ dnet:countries @=@ NP @=@ Nepal -dnet:countries @=@ dnet:countries @=@ NL @=@ Netherlands -dnet:countries @=@ dnet:countries @=@ AN @=@ Netherlands Antilles -dnet:countries @=@ dnet:countries @=@ NC @=@ New Caledonia -dnet:countries @=@ dnet:countries @=@ NZ @=@ New Zealand -dnet:countries @=@ dnet:countries @=@ NI @=@ Nicaragua -dnet:countries @=@ dnet:countries @=@ NE @=@ Niger -dnet:countries @=@ dnet:countries @=@ NG @=@ Nigeria -dnet:countries @=@ dnet:countries @=@ NU @=@ Niue -dnet:countries @=@ dnet:countries @=@ NF @=@ Norfolk Island -dnet:countries @=@ dnet:countries @=@ MP @=@ Northern Mariana Islands -dnet:countries @=@ dnet:countries @=@ NO @=@ Norway -dnet:countries @=@ dnet:countries @=@ OC @=@ Oceania -dnet:countries @=@ dnet:countries @=@ OM @=@ Oman -dnet:countries @=@ dnet:countries @=@ PK @=@ Pakistan -dnet:countries @=@ dnet:countries @=@ PW @=@ Palau -dnet:countries @=@ dnet:countries @=@ PS @=@ Palestinian-administered areas -dnet:countries @=@ dnet:countries @=@ PA @=@ Panama -dnet:countries @=@ dnet:countries @=@ PG @=@ Papua New Guinea -dnet:countries @=@ dnet:countries @=@ PY @=@ Paraguay -dnet:countries @=@ dnet:countries @=@ PE @=@ Peru -dnet:countries @=@ dnet:countries @=@ PH @=@ Philippines -dnet:countries @=@ dnet:countries @=@ PN @=@ Pitcairn -dnet:countries @=@ dnet:countries @=@ PL @=@ Poland -dnet:countries @=@ dnet:countries @=@ PT @=@ Portugal -dnet:countries @=@ dnet:countries @=@ PR @=@ Puerto Rico -dnet:countries @=@ dnet:countries @=@ QA @=@ Qatar -dnet:countries @=@ dnet:countries @=@ RO @=@ Romania -dnet:countries @=@ dnet:countries @=@ RU @=@ Russian Federation -dnet:countries @=@ dnet:countries @=@ RW @=@ Rwanda -dnet:countries @=@ dnet:countries @=@ RE @=@ Réunion -dnet:countries @=@ dnet:countries @=@ SH @=@ Saint Helena, Ascension and Tristan da Cunha -dnet:countries @=@ dnet:countries @=@ KN @=@ Saint Kitts and Nevis -dnet:countries @=@ dnet:countries @=@ LC @=@ Saint Lucia -dnet:countries @=@ dnet:countries @=@ MF @=@ Saint Martin (French Part) -dnet:countries @=@ dnet:countries @=@ PM @=@ Saint Pierre and Miquelon -dnet:countries @=@ dnet:countries @=@ VC @=@ Saint Vincent and the Grenadines -dnet:countries @=@ dnet:countries @=@ BL @=@ Saint-Barthélemy -dnet:countries @=@ dnet:countries @=@ WS @=@ Samoa -dnet:countries @=@ dnet:countries @=@ SM @=@ San Marino -dnet:countries @=@ dnet:countries @=@ SA @=@ Saudi Arabia -dnet:countries @=@ dnet:countries @=@ SN @=@ Senegal -dnet:countries @=@ dnet:countries @=@ RS @=@ Serbia -dnet:countries @=@ dnet:countries @=@ SC @=@ Seychelles -dnet:countries @=@ dnet:countries @=@ SL @=@ Sierra Leone -dnet:countries @=@ dnet:countries @=@ SG @=@ Singapore -dnet:countries @=@ dnet:countries @=@ SX @=@ Sint Maarten (Dutch Part) -dnet:countries @=@ dnet:countries @=@ SK @=@ Slovakia -dnet:countries @=@ dnet:countries @=@ SI @=@ Slovenia -dnet:countries @=@ dnet:countries @=@ SB @=@ Solomon Islands -dnet:countries @=@ dnet:countries @=@ SO @=@ Somalia -dnet:countries @=@ dnet:countries @=@ ZA @=@ South Africa -dnet:countries @=@ dnet:countries @=@ GS @=@ South Georgia and the South Sandwich Islands -dnet:countries @=@ dnet:countries @=@ SS @=@ South Sudan -dnet:countries @=@ dnet:countries @=@ ES @=@ Spain -dnet:countries @=@ dnet:countries @=@ LK @=@ Sri Lanka -dnet:countries @=@ dnet:countries @=@ SD @=@ Sudan -dnet:countries @=@ dnet:countries @=@ SR @=@ Suriname -dnet:countries @=@ dnet:countries @=@ SJ @=@ Svalbard and Jan Mayen -dnet:countries @=@ dnet:countries @=@ SZ @=@ Swaziland -dnet:countries @=@ dnet:countries @=@ SE @=@ Sweden -dnet:countries @=@ dnet:countries @=@ CH @=@ Switzerland -dnet:countries @=@ dnet:countries @=@ SY @=@ Syrian Arab Republic -dnet:countries @=@ dnet:countries @=@ ST @=@ São Tomé and Príncipe -dnet:countries @=@ dnet:countries @=@ TW @=@ Taiwan -dnet:countries @=@ dnet:countries @=@ TJ @=@ Tajikistan -dnet:countries @=@ dnet:countries @=@ TZ @=@ Tanzania (United Republic of) -dnet:countries @=@ dnet:countries @=@ TH @=@ Thailand -dnet:countries @=@ dnet:countries @=@ TL @=@ Timor-Leste -dnet:countries @=@ dnet:countries @=@ TG @=@ Togo -dnet:countries @=@ dnet:countries @=@ TK @=@ Tokelau -dnet:countries @=@ dnet:countries @=@ TO @=@ Tonga -dnet:countries @=@ dnet:countries @=@ TT @=@ Trinidad and Tobago -dnet:countries @=@ dnet:countries @=@ TN @=@ Tunisia -dnet:countries @=@ dnet:countries @=@ TR @=@ Turkey -dnet:countries @=@ dnet:countries @=@ TM @=@ Turkmenistan -dnet:countries @=@ dnet:countries @=@ TC @=@ Turks and Caicos Islands -dnet:countries @=@ dnet:countries @=@ TV @=@ Tuvalu -dnet:countries @=@ dnet:countries @=@ UNKNOWN @=@ UNKNOWN -dnet:countries @=@ dnet:countries @=@ UG @=@ Uganda -dnet:countries @=@ dnet:countries @=@ UA @=@ Ukraine -dnet:countries @=@ dnet:countries @=@ AE @=@ United Arab Emirates -dnet:countries @=@ dnet:countries @=@ GB @=@ United Kingdom -dnet:countries @=@ dnet:countries @=@ US @=@ United States -dnet:countries @=@ dnet:countries @=@ UM @=@ United States Minor Outlying Islands -dnet:countries @=@ dnet:countries @=@ UY @=@ Uruguay -dnet:countries @=@ dnet:countries @=@ UZ @=@ Uzbekistan -dnet:countries @=@ dnet:countries @=@ VU @=@ Vanuatu -dnet:countries @=@ dnet:countries @=@ VE @=@ Venezuela -dnet:countries @=@ dnet:countries @=@ VN @=@ Viet Nam -dnet:countries @=@ dnet:countries @=@ VG @=@ Virgin Islands (British) -dnet:countries @=@ dnet:countries @=@ VI @=@ Virgin Islands, U.S. -dnet:countries @=@ dnet:countries @=@ WF @=@ Wallis and Futuna -dnet:countries @=@ dnet:countries @=@ EH @=@ Western Sahara -dnet:countries @=@ dnet:countries @=@ YE @=@ Yemen -dnet:countries @=@ dnet:countries @=@ YU @=@ Yugoslavia -dnet:countries @=@ dnet:countries @=@ ZM @=@ Zambia -dnet:countries @=@ dnet:countries @=@ ZW @=@ Zimbabwe -dnet:countries @=@ dnet:countries @=@ AX @=@ Åland Islands -ec:funding_relations @=@ ec:funding_relations @=@ ec:hasframeworkprogram @=@ hasframeworkprogram -ec:funding_relations @=@ ec:funding_relations @=@ ec:hasprogram @=@ hasprogram -ec:funding_relations @=@ ec:funding_relations @=@ ec:hasspecificprogram @=@ hasspecificprogram -dnet:dataCite_date @=@ dnet:dataCite_date @=@ available @=@ available -dnet:dataCite_date @=@ dnet:dataCite_date @=@ copyrighted @=@ copyrighted -dnet:dataCite_date @=@ dnet:dataCite_date @=@ created @=@ created -dnet:dataCite_date @=@ dnet:dataCite_date @=@ endDate @=@ endDate -dnet:dataCite_date @=@ dnet:dataCite_date @=@ issued @=@ issued -dnet:dataCite_date @=@ dnet:dataCite_date @=@ startDate @=@ startDate -dnet:dataCite_date @=@ dnet:dataCite_date @=@ submitted @=@ submitted -dnet:dataCite_date @=@ dnet:dataCite_date @=@ UNKNOWN @=@ UNKNOWN -dnet:dataCite_date @=@ dnet:dataCite_date @=@ updated @=@ updated -dnet:dataCite_date @=@ dnet:dataCite_date @=@ valid @=@ valid -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire2.0 @=@ OpenAIRE 2.0 (EC funding) -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ driver-openaire2.0 @=@ OpenAIRE 2.0+ (DRIVER OA, EC funding) -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire3.0 @=@ OpenAIRE 3.0 (OA, funding) -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ driver @=@ OpenAIRE Basic (DRIVER OA) -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire-cris_1.1 @=@ OpenAIRE CRIS v1.1 -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire2.0_data @=@ OpenAIRE Data (funded, referenced datasets) -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire-pub_4.0 @=@ OpenAIRE PubRepos v4.0 -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ hostedBy @=@ collected from a compatible aggregator -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ UNKNOWN @=@ not available -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ native @=@ proprietary -dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ notCompatible @=@ under validation -dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ accessionNumber @=@ accessionNumber -dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ software @=@ software -dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ dataset @=@ dataset -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ 171 @=@ Article 171 of the Treaty -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ BSG @=@ Research for the benefit of specific groups -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CIP-EIP-TN @=@ CIP-Eco-Innovation - CIP-Thematic Network -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CP @=@ Collaborative project -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CP-CSA @=@ Combination of CP & CSA -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CSA @=@ Coordination and support action -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ ERC @=@ Support for frontier research (ERC) -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ MC @=@ Support for training and career development of researchers (Marie Curie) -ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ NoE @=@ Network of Excellence -fct:funding_relations @=@ fct:funding_relations @=@ fct:hasParentFunding @=@ fct:hasParentFunding -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-ADG @=@ Advanced Grant -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-CSA @=@ Bio-based Industries Coordination and Support action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-IA-DEMO @=@ Bio-based Industries Innovation action - Demonstration -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-IA-FLAG @=@ Bio-based Industries Innovation action - Flagship -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-RIA @=@ Bio-based Industries Research and Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-CAR @=@ CAR – Career Restart panel -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-EJP @=@ COFUND (European Joint Programme) -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-PCP @=@ COFUND (PCP) -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-PPI @=@ COFUND (PPI) -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-CSA @=@ CS2 Coordination and Support action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-IA @=@ CS2 Innovation Action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-RIA @=@ CS2 Research and Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CSA-LS @=@ CSA Lump sum -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-COG @=@ Consolidator Grant -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-CSA @=@ Coordination & support action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CSA @=@ Coordination and support action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-COFUND-DP @=@ Doctoral programmes -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-CSA @=@ ECSEL Coordination & Support action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-IA @=@ ECSEL Innovation Action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-RIA @=@ ECSEL Research and Innovation Actions -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERA-NET-Cofund @=@ ERA-NET Cofund -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-POC-LS @=@ ERC Proof of Concept Lump Sum Pilot -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-LVG @=@ ERC low value grant -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ H2020-EEN-SGA @=@ Enterprise Europe Network - Specific Grant Agreement -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-EID @=@ European Industrial Doctorates -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-EJD @=@ European Joint Doctorates -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-ETN @=@ European Training Networks -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-IA @=@ FCH2 Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-RIA @=@ FCH2 Research and Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-COFUND-FP @=@ Fellowship programmes -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-GF @=@ Global Fellowships -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IMI2-CSA @=@ IMI2 Coordination & support action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IMI2-RIA @=@ IMI2 Research and Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-IA-LS @=@ Innovation Action Lump-Sum -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IA-LS @=@ Innovation Action Lump-Sum -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IA @=@ Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-IA @=@ Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ PCP @=@ Pre-Commercial Procurement -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-POC @=@ Proof of Concept Grant -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ PPI @=@ Public Procurement of Innovative Solutions -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-RI @=@ RI – Reintegration panel -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-RISE @=@ RISE -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-RIA-LS @=@ Research and Innovation Action Lump-Sum| -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-RIA @=@ Research and Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ RIA @=@ Research and Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ RIA-LS @=@ Research and Innovation action Lump Sum -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-CSA @=@ SESAR: Coordination and Support Action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-IA @=@ SESAR: Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-RIA @=@ SESAR: Research and Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SGA-RIA @=@ SGA Research and Innovation action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-1 @=@ SME instrument phase 1 -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-2 @=@ SME instrument phase 2 -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-CSA @=@ Shift2Rail - Coordination and Support action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-SE @=@ Society and Enterprise panel -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SGA-CSA @=@ Specific Grant agreement and Coordination and Support Action -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-ST @=@ Standard EF -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-STG @=@ Starting Grant -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-SyG @=@ Synergy grant -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-SNLS @=@ Grant to identified beneficiary - Coordination and support actions (MSCA-Special Needs lump sum) -ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-2b @=@ SME Instrument (grant only and blended finance) -dnet:dataCite_title @=@ dnet:dataCite_title @=@ alternative title @=@ alternative title -dnet:dataCite_title @=@ dnet:dataCite_title @=@ main title @=@ main title -dnet:dataCite_title @=@ dnet:dataCite_title @=@ subtitle @=@ subtitle -dnet:dataCite_title @=@ dnet:dataCite_title @=@ translated title @=@ translated title -nsf:contractTypes @=@ NSF Contract Types @=@ BOA/Task Order @=@ BOA/Task Order -nsf:contractTypes @=@ NSF Contract Types @=@ Continuing grant @=@ Continuing grant -nsf:contractTypes @=@ NSF Contract Types @=@ Contract @=@ Contract -nsf:contractTypes @=@ NSF Contract Types @=@ Contract Interagency Agreement @=@ Contract Interagency Agreement -nsf:contractTypes @=@ NSF Contract Types @=@ Cooperative Agreement @=@ Cooperative Agreement -nsf:contractTypes @=@ NSF Contract Types @=@ Fellowship @=@ Fellowship -nsf:contractTypes @=@ NSF Contract Types @=@ Fixed Price Award @=@ Fixed Price Award -nsf:contractTypes @=@ NSF Contract Types @=@ Interagency Agreement @=@ Interagency Agreement -nsf:contractTypes @=@ NSF Contract Types @=@ Intergovernmental Personnel Award @=@ Intergovernmental Personnel Award -nsf:contractTypes @=@ NSF Contract Types @=@ Personnel Agreement @=@ Personnel Agreement -nsf:contractTypes @=@ NSF Contract Types @=@ Standard Grant @=@ Standard Grant -nsf:contractTypes @=@ NSF Contract Types @=@ GAA @=@ GAA -wt:funding_relations @=@ wt:funding_relations @=@ wt:hasParentFunding @=@ wt:hasParentFunding +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::EuropePMC @=@ file::EuropePMC +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::PDF @=@ file::PDF +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::WoS @=@ file::WoS +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ metadata @=@ metadata +dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::hybrid @=@ file::hybrid +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:cris @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:actionset:orcidworks-no-doi @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:infospace @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:aggregator @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:datasetarchive @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:actionset @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:entityregistry @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:repository @=@ Harvested +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:aggregator @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:subject @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:zenodocommunity @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ iis @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:entityregistry @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:organization @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:infospace @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:dedup @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:datasource @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ propagation:project:semrel @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:cris @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:repository @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:datasetarchive @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ community:semrel @=@ Inferred by OpenAIRE +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim:pid @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:insert @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim:search @=@ Linked by user +dnet:provenanceActions @=@ dnet:provenanceActions @=@ UNKNOWN @=@ UNKNOWN +dnet:provenanceActions @=@ dnet:provenanceActions @=@ country:instrepos @=@ Inferred by OpenAIRE dnet:access_modes @=@ dnet:access_modes @=@ 12MONTHS @=@ 12 Months Embargo dnet:access_modes @=@ dnet:access_modes @=@ 6MONTHS @=@ 6 Months Embargo dnet:access_modes @=@ dnet:access_modes @=@ CLOSED @=@ Closed Access @@ -418,10 +43,140 @@ dnet:access_modes @=@ dnet:access_modes @=@ OPEN SOURCE @=@ Open Source dnet:access_modes @=@ dnet:access_modes @=@ OTHER @=@ Other dnet:access_modes @=@ dnet:access_modes @=@ RESTRICTED @=@ Restricted dnet:access_modes @=@ dnet:access_modes @=@ UNKNOWN @=@ not available -dnet:result_typologies @=@ dnet:result_typologies @=@ dataset @=@ dataset -dnet:result_typologies @=@ dnet:result_typologies @=@ other @=@ other -dnet:result_typologies @=@ dnet:result_typologies @=@ publication @=@ publication -dnet:result_typologies @=@ dnet:result_typologies @=@ software @=@ software +fct:funding_typologies @=@ fct:funding_typologies @=@ fct:program @=@ fct:program +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire2.0 @=@ OpenAIRE 2.0 (EC funding) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire3.0 @=@ OpenAIRE 3.0 (OA, funding) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ driver @=@ OpenAIRE Basic (DRIVER OA) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire-cris_1.1 @=@ OpenAIRE CRIS v1.1 +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire2.0_data @=@ OpenAIRE Data (funded, referenced datasets) +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire-pub_4.0 @=@ OpenAIRE PubRepos v4.0 +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ hostedBy @=@ collected from a compatible aggregator +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ files @=@ files +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ native @=@ native +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ UNKNOWN @=@ not available +dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ notCompatible @=@ under validation +dnet:dataCite_date @=@ dnet:dataCite_date @=@ UNKNOWN @=@ UNKNOWN +dnet:dataCite_date @=@ dnet:dataCite_date @=@ available @=@ available +dnet:dataCite_date @=@ dnet:dataCite_date @=@ copyrighted @=@ copyrighted +dnet:dataCite_date @=@ dnet:dataCite_date @=@ created @=@ created +dnet:dataCite_date @=@ dnet:dataCite_date @=@ endDate @=@ endDate +dnet:dataCite_date @=@ dnet:dataCite_date @=@ issued @=@ issued +dnet:dataCite_date @=@ dnet:dataCite_date @=@ startDate @=@ startDate +dnet:dataCite_date @=@ dnet:dataCite_date @=@ submitted @=@ submitted +dnet:dataCite_date @=@ dnet:dataCite_date @=@ updated @=@ updated +dnet:dataCite_date @=@ dnet:dataCite_date @=@ valid @=@ valid +dnet:dataCite_date @=@ dnet:dataCite_date @=@ published-print @=@ published-print +dnet:dataCite_date @=@ dnet:dataCite_date @=@ published-online @=@ published-online +dnet:dataCite_date @=@ dnet:dataCite_date @=@ accepted @=@ accepted +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ crissystem @=@ CRIS System +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ datarepository::unknown @=@ Data Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::datarepository @=@ Data Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::projects @=@ Funder database +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ infospace @=@ Information Space +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::institutional @=@ Institutional Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::institutional @=@ Institutional Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::journal @=@ Journal +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::journals @=@ Journal Aggregator/Publisher +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::mock @=@ Other +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubscatalogue::unknown @=@ Publication Catalogue +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::unknown @=@ Publication Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::unknown @=@ Publication Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry @=@ Registry +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::repositories @=@ Registry of repositories +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::products @=@ Registry of research products +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::researchers @=@ Registry of researchers +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::organizations @=@ Registry of organizations +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ scholarcomminfra @=@ Scholarly Comm. Infrastructure +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ softwarerepository @=@ Software Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::thematic @=@ Thematic Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ websource @=@ Web Source +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::softwarerepository @=@ Software Repository Aggregator +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ orprepository @=@ Repository +dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ researchgraph @=@ Research Graph +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ACM @=@ ACM Computing Classification System +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ agrovoc @=@ AGROVOC +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ bicssc @=@ BIC standard subject categories +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ DFG @=@ DFG Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ddc @=@ Dewey Decimal Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ nsf:fieldOfApplication @=@ Field of Application (NSF) +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ gok @=@ Göttingen Online Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ec:h2020topics @=@ Horizon 2020 Topics +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ IPC @=@ International Patent Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ jel @=@ JEL Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ lcsh @=@ Library of Congress Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ msc @=@ Mathematics Subject Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ mesheuropmc @=@ Medical Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ mesh @=@ Medical Subject Headings +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ bk @=@ Nederlandse basisclassificatie +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ dnet:od_subjects @=@ OpenDOAR subjects +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ocis @=@ Optics Classification and Indexing Scheme +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ pacs @=@ Physics and Astronomy Classification Scheme +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ rvk @=@ Regensburger Verbundklassifikation +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ UNKNOWN @=@ UNKNOWN +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ udc @=@ Universal Decimal Classification +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ wos @=@ Web of Science Subject Areas +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ arxiv @=@ arXiv +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ keyword @=@ keyword +dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ MAG @=@ Microsoft Academic Graph classification +fct:contractTypes @=@ fct:contractTypes @=@ UNKNOWN @=@ UNKNOWN +dnet:publication_resource @=@ dnet:publication_resource @=@ 0018 @=@ Annotation +dnet:publication_resource @=@ dnet:publication_resource @=@ 0001 @=@ Article +dnet:publication_resource @=@ dnet:publication_resource @=@ 0033 @=@ Audiovisual +dnet:publication_resource @=@ dnet:publication_resource @=@ 0008 @=@ Bachelor thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0046 @=@ Bioentity +dnet:publication_resource @=@ dnet:publication_resource @=@ 0002 @=@ Book +dnet:publication_resource @=@ dnet:publication_resource @=@ 0037 @=@ Clinical Trial +dnet:publication_resource @=@ dnet:publication_resource @=@ 0022 @=@ Collection +dnet:publication_resource @=@ dnet:publication_resource @=@ 0004 @=@ Conference object +dnet:publication_resource @=@ dnet:publication_resource @=@ 0005 @=@ Contribution for newspaper or weekly magazine +dnet:publication_resource @=@ dnet:publication_resource @=@ 0045 @=@ Data Management Plan +dnet:publication_resource @=@ dnet:publication_resource @=@ 0031 @=@ Data Paper +dnet:publication_resource @=@ dnet:publication_resource @=@ 0021 @=@ Dataset +dnet:publication_resource @=@ dnet:publication_resource @=@ 0006 @=@ Doctoral thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0023 @=@ Event +dnet:publication_resource @=@ dnet:publication_resource @=@ 0009 @=@ External research report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0024 @=@ Film +dnet:publication_resource @=@ dnet:publication_resource @=@ 0025 @=@ Image +dnet:publication_resource @=@ dnet:publication_resource @=@ 0026 @=@ InteractiveResource +dnet:publication_resource @=@ dnet:publication_resource @=@ 0011 @=@ Internal report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0043 @=@ Journal +dnet:publication_resource @=@ dnet:publication_resource @=@ 0010 @=@ Lecture +dnet:publication_resource @=@ dnet:publication_resource @=@ 0007 @=@ Master thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0027 @=@ Model +dnet:publication_resource @=@ dnet:publication_resource @=@ 0012 @=@ Newsletter +dnet:publication_resource @=@ dnet:publication_resource @=@ 0020 @=@ Other ORP type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0039 @=@ Other dataset type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0038 @=@ Other literature type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0040 @=@ Other software type +dnet:publication_resource @=@ dnet:publication_resource @=@ 0013 @=@ Part of book or chapter of book +dnet:publication_resource @=@ dnet:publication_resource @=@ 0019 @=@ Patent +dnet:publication_resource @=@ dnet:publication_resource @=@ 0028 @=@ PhysicalObject +dnet:publication_resource @=@ dnet:publication_resource @=@ 0016 @=@ Preprint +dnet:publication_resource @=@ dnet:publication_resource @=@ 0034 @=@ Project deliverable +dnet:publication_resource @=@ dnet:publication_resource @=@ 0035 @=@ Project milestone +dnet:publication_resource @=@ dnet:publication_resource @=@ 0036 @=@ Project proposal +dnet:publication_resource @=@ dnet:publication_resource @=@ 0017 @=@ Report +dnet:publication_resource @=@ dnet:publication_resource @=@ 0014 @=@ Research +dnet:publication_resource @=@ dnet:publication_resource @=@ 0015 @=@ Review +dnet:publication_resource @=@ dnet:publication_resource @=@ 0029 @=@ Software +dnet:publication_resource @=@ dnet:publication_resource @=@ 0032 @=@ Software Paper +dnet:publication_resource @=@ dnet:publication_resource @=@ 0030 @=@ Sound +dnet:publication_resource @=@ dnet:publication_resource @=@ 0044 @=@ Thesis +dnet:publication_resource @=@ dnet:publication_resource @=@ 0000 @=@ Unknown +dnet:publication_resource @=@ dnet:publication_resource @=@ 0042 @=@ Virtual Appliance +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:frameworkprogram @=@ frameworkprogram +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:program @=@ program +ec:funding_typologies @=@ ec:funding_typologies @=@ ec:specificprogram @=@ specificprogram +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ 171 @=@ Article 171 of the Treaty +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ BSG @=@ Research for the benefit of specific groups +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CIP-EIP-TN @=@ CIP-Eco-Innovation - CIP-Thematic Network +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CP @=@ Collaborative project +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CP-CSA @=@ Combination of CP & CSA +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ CSA @=@ Coordination and support action +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ ERC @=@ Support for frontier research (ERC) +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ MC @=@ Support for training and career development of researchers (Marie Curie) +ec:FP7contractTypes @=@ ec:FP7contractTypes @=@ NoE @=@ Network of Excellence +wt:funding_relations @=@ wt:funding_relations @=@ wt:hasParentFunding @=@ wt:hasParentFunding dnet:languages @=@ dnet:languages @=@ abk @=@ Abkhazian dnet:languages @=@ dnet:languages @=@ ace @=@ Achinese dnet:languages @=@ dnet:languages @=@ ach @=@ Acoli @@ -837,62 +592,389 @@ dnet:languages @=@ dnet:languages @=@ zha @=@ Zhuang; Chuang dnet:languages @=@ dnet:languages @=@ zul @=@ Zulu dnet:languages @=@ dnet:languages @=@ zun @=@ Zuni dnet:languages @=@ dnet:languages @=@ sga @=@ old Irish -dnet:pid_types @=@ dnet:pid_types @=@ GeoPass @=@ Geographic Location-Password Scheme -dnet:pid_types @=@ dnet:pid_types @=@ GBIF @=@ Global Biodiversity Information Facility -dnet:pid_types @=@ dnet:pid_types @=@ ISNI @=@ International Standard Name Identifier -dnet:pid_types @=@ dnet:pid_types @=@ oai @=@ Open Archives Initiative -dnet:pid_types @=@ dnet:pid_types @=@ orcid @=@ Open Researcher and Contributor ID -dnet:pid_types @=@ dnet:pid_types @=@ PANGAEA @=@ PANGAEA -dnet:pid_types @=@ dnet:pid_types @=@ UNKNOWN @=@ UNKNOWN -dnet:pid_types @=@ dnet:pid_types @=@ VIAF @=@ Virtual International Authority File -dnet:pid_types @=@ dnet:pid_types @=@ arXiv @=@ arXiv -dnet:pid_types @=@ dnet:pid_types @=@ doi @=@ doi -dnet:pid_types @=@ dnet:pid_types @=@ info:eu-repo/dai @=@ info:eu-repo/dai -dnet:pid_types @=@ dnet:pid_types @=@ orcidworkid @=@ orcid workid -dnet:pid_types @=@ dnet:pid_types @=@ pmc @=@ pmc -dnet:pid_types @=@ dnet:pid_types @=@ pmid @=@ pmid -dnet:pid_types @=@ dnet:pid_types @=@ urn @=@ urn -dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::EuropePMC @=@ file::EuropePMC -dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::PDF @=@ file::PDF -dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::WoS @=@ file::WoS -dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ metadata @=@ metadata -dnet:content_description_typologies @=@ D-Net Content Description Typologies @=@ file::hybrid @=@ file::hybrid -fct:funding_typologies @=@ fct:funding_typologies @=@ fct:program @=@ fct:program -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ACM @=@ ACM Computing Classification System -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ agrovoc @=@ AGROVOC -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ bicssc @=@ BIC standard subject categories -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ DFG @=@ DFG Classification -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ddc @=@ Dewey Decimal Classification -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ nsf:fieldOfApplication @=@ Field of Application (NSF) -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ gok @=@ Göttingen Online Classification -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ec:h2020topics @=@ Horizon 2020 Topics -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ jel @=@ JEL Classification -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ lcsh @=@ Library of Congress Subject Headings -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ msc @=@ Mathematics Subject Classification -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ mesheuropmc @=@ Medical Subject Headings -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ mesh @=@ Medical Subject Headings -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ bk @=@ Nederlandse basisclassificatie -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ dnet:od_subjects @=@ OpenDOAR subjects -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ ocis @=@ Optics Classification and Indexing Scheme -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ pacs @=@ Physics and Astronomy Classification Scheme -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ rvk @=@ Regensburger Verbundklassifikation -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ UNKNOWN @=@ UNKNOWN -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ wos @=@ Web of Science Subject Areas -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ arxiv @=@ arXiv -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ keyword @=@ keyword -dnet:subject_classification_typologies @=@ dnet:subject_classification_typologies @=@ udc @=@ Universal Decimal Classification +nsf:contractTypes @=@ NSF Contract Types @=@ BOA/Task Order @=@ BOA/Task Order +nsf:contractTypes @=@ NSF Contract Types @=@ Continuing grant @=@ Continuing grant +nsf:contractTypes @=@ NSF Contract Types @=@ Contract @=@ Contract +nsf:contractTypes @=@ NSF Contract Types @=@ Contract Interagency Agreement @=@ Contract Interagency Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Cooperative Agreement @=@ Cooperative Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Fellowship @=@ Fellowship +nsf:contractTypes @=@ NSF Contract Types @=@ Fixed Price Award @=@ Fixed Price Award +nsf:contractTypes @=@ NSF Contract Types @=@ GAA @=@ GAA +nsf:contractTypes @=@ NSF Contract Types @=@ Interagency Agreement @=@ Interagency Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Intergovernmental Personnel Award @=@ Intergovernmental Personnel Award +nsf:contractTypes @=@ NSF Contract Types @=@ Personnel Agreement @=@ Personnel Agreement +nsf:contractTypes @=@ NSF Contract Types @=@ Standard Grant @=@ Standard Grant +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasframeworkprogram @=@ hasframeworkprogram +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasprogram @=@ hasprogram +ec:funding_relations @=@ ec:funding_relations @=@ ec:hasspecificprogram @=@ hasspecificprogram +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ UNKNOWN @=@ UNKNOWN +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ collection @=@ collection +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ dataset @=@ dataset +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ event @=@ event +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ film @=@ film +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ image @=@ image +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ interactiveResource @=@ interactiveResource +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ model @=@ model +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ physicalObject @=@ physicalObject +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ service @=@ service +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ software @=@ software +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ sound @=@ sound +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ text @=@ text +dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ clinicalTrial @=@ Clinical trial +dnet:dataCite_title @=@ dnet:dataCite_title @=@ alternative title @=@ alternative title +dnet:dataCite_title @=@ dnet:dataCite_title @=@ main title @=@ main title +dnet:dataCite_title @=@ dnet:dataCite_title @=@ subtitle @=@ subtitle +dnet:dataCite_title @=@ dnet:dataCite_title @=@ translated title @=@ translated title +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsCitedBy @=@ IsCitedBy +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsNewVersionOf @=@ IsNewVersionOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsPartOf @=@ IsPartOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsPreviousVersionOf @=@ IsPreviousVersionOf +datacite:relation_typologies @=@ datacite:relation_typologies @=@ IsReferencedBy @=@ IsReferencedBy +datacite:relation_typologies @=@ datacite:relation_typologies @=@ References @=@ References +datacite:relation_typologies @=@ datacite:relation_typologies @=@ UNKNOWN @=@ UNKNOWN +dnet:result_typologies @=@ dnet:result_typologies @=@ dataset @=@ dataset +dnet:result_typologies @=@ dnet:result_typologies @=@ other @=@ other +dnet:result_typologies @=@ dnet:result_typologies @=@ publication @=@ publication +dnet:result_typologies @=@ dnet:result_typologies @=@ software @=@ software +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-ADG @=@ Advanced Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-CSA @=@ Bio-based Industries Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-IA-DEMO @=@ Bio-based Industries Innovation action - Demonstration +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-IA-FLAG @=@ Bio-based Industries Innovation action - Flagship +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ BBI-RIA @=@ Bio-based Industries Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-CAR @=@ CAR – Career Restart panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-EJP @=@ COFUND (European Joint Programme) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-PCP @=@ COFUND (PCP) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ COFUND-PPI @=@ COFUND (PPI) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-CSA @=@ CS2 Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-IA @=@ CS2 Innovation Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CS2-RIA @=@ CS2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CSA-LS @=@ CSA Lump sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-COG @=@ Consolidator Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-CSA @=@ Coordination & support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ CSA @=@ Coordination and support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-COFUND-DP @=@ Doctoral programmes +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-CSA @=@ ECSEL Coordination & Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-IA @=@ ECSEL Innovation Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ECSEL-RIA @=@ ECSEL Research and Innovation Actions +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERA-NET-Cofund @=@ ERA-NET Cofund +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-POC-LS @=@ ERC Proof of Concept Lump Sum Pilot +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-SyG @=@ ERC Synergy Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-LVG @=@ ERC low value grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ H2020-EEN-SGA @=@ Enterprise Europe Network - Specific Grant Agreement +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-EID @=@ European Industrial Doctorates +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-EJD @=@ European Joint Doctorates +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-ITN-ETN @=@ European Training Networks +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-IA @=@ FCH2 Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ FCH2-RIA @=@ FCH2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-COFUND-FP @=@ Fellowship programmes +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-GF @=@ Global Fellowships +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IMI2-CSA @=@ IMI2 Coordination & support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IMI2-RIA @=@ IMI2 Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-IA-LS @=@ Innovation Action Lump-Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IA-LS @=@ Innovation Action Lump-Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ IA @=@ Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-IA @=@ Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ PCP @=@ Pre-Commercial Procurement +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-POC @=@ Proof of Concept Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ PPI @=@ Public Procurement of Innovative Solutions +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-RI @=@ RI – Reintegration panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-RISE @=@ RISE +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-RIA-LS @=@ Research and Innovation Action Lump-Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-RIA @=@ Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ RIA @=@ Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ RIA-LS @=@ Research and Innovation action Lump Sum +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-CSA @=@ SESAR: Coordination and Support Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-IA @=@ SESAR: Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SESAR-RIA @=@ SESAR: Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SGA-RIA @=@ SGA Research and Innovation action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-2b @=@ SME Instrument (grant only and blended finance) +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-1 @=@ SME instrument phase 1 +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SME-2 @=@ SME instrument phase 2 +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ Shift2Rail-CSA @=@ Shift2Rail - Coordination and Support action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-SE @=@ Society and Enterprise panel +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ SGA-CSA @=@ Specific Grant agreement and Coordination and Support Action +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-IF-EF-ST @=@ Standard EF +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ ERC-STG @=@ Starting Grant +ec:h2020toas @=@ Horizon 2020 - Type of Actions @=@ MSCA-SNLS @=@ Grant to identified beneficiary - Coordination and support actions (MSCA-Special Needs lump sum) +wt:contractTypes @=@ wt:contractTypes @=@ UNKNOWN @=@ UNKNOWN +dnet:countries @=@ dnet:countries @=@ AF @=@ Afghanistan +dnet:countries @=@ dnet:countries @=@ AL @=@ Albania +dnet:countries @=@ dnet:countries @=@ DZ @=@ Algeria +dnet:countries @=@ dnet:countries @=@ AS @=@ American Samoa +dnet:countries @=@ dnet:countries @=@ AD @=@ Andorra +dnet:countries @=@ dnet:countries @=@ AO @=@ Angola +dnet:countries @=@ dnet:countries @=@ AI @=@ Anguilla +dnet:countries @=@ dnet:countries @=@ AQ @=@ Antarctica +dnet:countries @=@ dnet:countries @=@ AG @=@ Antigua and Barbuda +dnet:countries @=@ dnet:countries @=@ AR @=@ Argentina +dnet:countries @=@ dnet:countries @=@ AM @=@ Armenia +dnet:countries @=@ dnet:countries @=@ AW @=@ Aruba +dnet:countries @=@ dnet:countries @=@ AU @=@ Australia +dnet:countries @=@ dnet:countries @=@ AT @=@ Austria +dnet:countries @=@ dnet:countries @=@ AZ @=@ Azerbaijan +dnet:countries @=@ dnet:countries @=@ BS @=@ Bahamas +dnet:countries @=@ dnet:countries @=@ BH @=@ Bahrain +dnet:countries @=@ dnet:countries @=@ BD @=@ Bangladesh +dnet:countries @=@ dnet:countries @=@ BB @=@ Barbados +dnet:countries @=@ dnet:countries @=@ BY @=@ Belarus +dnet:countries @=@ dnet:countries @=@ BE @=@ Belgium +dnet:countries @=@ dnet:countries @=@ BZ @=@ Belize +dnet:countries @=@ dnet:countries @=@ BJ @=@ Benin +dnet:countries @=@ dnet:countries @=@ BM @=@ Bermuda +dnet:countries @=@ dnet:countries @=@ BT @=@ Bhutan +dnet:countries @=@ dnet:countries @=@ BO @=@ Bolivia +dnet:countries @=@ dnet:countries @=@ BQ @=@ Bonaire, Sint Eustatius and Saba +dnet:countries @=@ dnet:countries @=@ BA @=@ Bosnia and Herzegovina +dnet:countries @=@ dnet:countries @=@ BW @=@ Botswana +dnet:countries @=@ dnet:countries @=@ BV @=@ Bouvet Island +dnet:countries @=@ dnet:countries @=@ BR @=@ Brazil +dnet:countries @=@ dnet:countries @=@ IO @=@ British Indian Ocean Territory +dnet:countries @=@ dnet:countries @=@ BN @=@ Brunei Darussalam +dnet:countries @=@ dnet:countries @=@ BG @=@ Bulgaria +dnet:countries @=@ dnet:countries @=@ BF @=@ Burkina Faso +dnet:countries @=@ dnet:countries @=@ BI @=@ Burundi +dnet:countries @=@ dnet:countries @=@ KH @=@ Cambodia +dnet:countries @=@ dnet:countries @=@ CM @=@ Cameroon +dnet:countries @=@ dnet:countries @=@ CA @=@ Canada +dnet:countries @=@ dnet:countries @=@ CV @=@ Cape Verde +dnet:countries @=@ dnet:countries @=@ KY @=@ Cayman Islands +dnet:countries @=@ dnet:countries @=@ CF @=@ Central African Republic +dnet:countries @=@ dnet:countries @=@ TD @=@ Chad +dnet:countries @=@ dnet:countries @=@ CL @=@ Chile +dnet:countries @=@ dnet:countries @=@ CN @=@ China (People's Republic of) +dnet:countries @=@ dnet:countries @=@ CX @=@ Christmas Island +dnet:countries @=@ dnet:countries @=@ CC @=@ Cocos (Keeling) Islands +dnet:countries @=@ dnet:countries @=@ CO @=@ Colombia +dnet:countries @=@ dnet:countries @=@ KM @=@ Comoros +dnet:countries @=@ dnet:countries @=@ CG @=@ Congo +dnet:countries @=@ dnet:countries @=@ CD @=@ Congo (Democratic Republic of) +dnet:countries @=@ dnet:countries @=@ CK @=@ Cook Islands +dnet:countries @=@ dnet:countries @=@ CR @=@ Costa Rica +dnet:countries @=@ dnet:countries @=@ CI @=@ Cote d'Ivoire +dnet:countries @=@ dnet:countries @=@ HR @=@ Croatia +dnet:countries @=@ dnet:countries @=@ CU @=@ Cuba +dnet:countries @=@ dnet:countries @=@ CW @=@ Curaçao +dnet:countries @=@ dnet:countries @=@ CY @=@ Cyprus +dnet:countries @=@ dnet:countries @=@ CZ @=@ Czech Republic +dnet:countries @=@ dnet:countries @=@ DK @=@ Denmark +dnet:countries @=@ dnet:countries @=@ DJ @=@ Djibouti +dnet:countries @=@ dnet:countries @=@ DM @=@ Dominica +dnet:countries @=@ dnet:countries @=@ DO @=@ Dominican Republic +dnet:countries @=@ dnet:countries @=@ EC @=@ Ecuador +dnet:countries @=@ dnet:countries @=@ EG @=@ Egypt +dnet:countries @=@ dnet:countries @=@ SV @=@ El Salvador +dnet:countries @=@ dnet:countries @=@ GQ @=@ Equatorial Guinea +dnet:countries @=@ dnet:countries @=@ ER @=@ Eritrea +dnet:countries @=@ dnet:countries @=@ EE @=@ Estonia +dnet:countries @=@ dnet:countries @=@ ET @=@ Ethiopia +dnet:countries @=@ dnet:countries @=@ EU @=@ European Union +dnet:countries @=@ dnet:countries @=@ FK @=@ Falkland Islands (Malvinas) +dnet:countries @=@ dnet:countries @=@ FO @=@ Faroe Islands +dnet:countries @=@ dnet:countries @=@ FJ @=@ Fiji +dnet:countries @=@ dnet:countries @=@ FI @=@ Finland +dnet:countries @=@ dnet:countries @=@ MK @=@ Former Yugoslav Republic of Macedonia +dnet:countries @=@ dnet:countries @=@ FR @=@ France +dnet:countries @=@ dnet:countries @=@ GF @=@ French Guiana +dnet:countries @=@ dnet:countries @=@ PF @=@ French Polynesia +dnet:countries @=@ dnet:countries @=@ TF @=@ French Southern Territories +dnet:countries @=@ dnet:countries @=@ GA @=@ Gabon +dnet:countries @=@ dnet:countries @=@ GM @=@ Gambia +dnet:countries @=@ dnet:countries @=@ GE @=@ Georgia +dnet:countries @=@ dnet:countries @=@ DE @=@ Germany +dnet:countries @=@ dnet:countries @=@ GH @=@ Ghana +dnet:countries @=@ dnet:countries @=@ GI @=@ Gibraltar +dnet:countries @=@ dnet:countries @=@ GR @=@ Greece +dnet:countries @=@ dnet:countries @=@ GL @=@ Greenland +dnet:countries @=@ dnet:countries @=@ GD @=@ Grenada +dnet:countries @=@ dnet:countries @=@ GP @=@ Guadeloupe +dnet:countries @=@ dnet:countries @=@ GU @=@ Guam +dnet:countries @=@ dnet:countries @=@ GT @=@ Guatemala +dnet:countries @=@ dnet:countries @=@ GG @=@ Guernsey +dnet:countries @=@ dnet:countries @=@ GN @=@ Guinea +dnet:countries @=@ dnet:countries @=@ GW @=@ Guinea-Bissau +dnet:countries @=@ dnet:countries @=@ GY @=@ Guyana +dnet:countries @=@ dnet:countries @=@ HT @=@ Haiti +dnet:countries @=@ dnet:countries @=@ HM @=@ Heard Island and McDonald Islands +dnet:countries @=@ dnet:countries @=@ VA @=@ Holy See (Vatican City State) +dnet:countries @=@ dnet:countries @=@ HN @=@ Honduras +dnet:countries @=@ dnet:countries @=@ HK @=@ Hong Kong +dnet:countries @=@ dnet:countries @=@ HU @=@ Hungary +dnet:countries @=@ dnet:countries @=@ IS @=@ Iceland +dnet:countries @=@ dnet:countries @=@ IN @=@ India +dnet:countries @=@ dnet:countries @=@ ID @=@ Indonesia +dnet:countries @=@ dnet:countries @=@ IR @=@ Iran (Islamic Republic of) +dnet:countries @=@ dnet:countries @=@ IQ @=@ Iraq +dnet:countries @=@ dnet:countries @=@ IE @=@ Ireland +dnet:countries @=@ dnet:countries @=@ IM @=@ Isle of Man +dnet:countries @=@ dnet:countries @=@ IL @=@ Israel +dnet:countries @=@ dnet:countries @=@ IT @=@ Italy +dnet:countries @=@ dnet:countries @=@ JM @=@ Jamaica +dnet:countries @=@ dnet:countries @=@ JP @=@ Japan +dnet:countries @=@ dnet:countries @=@ JE @=@ Jersey +dnet:countries @=@ dnet:countries @=@ JO @=@ Jordan +dnet:countries @=@ dnet:countries @=@ KZ @=@ Kazakhstan +dnet:countries @=@ dnet:countries @=@ KE @=@ Kenya +dnet:countries @=@ dnet:countries @=@ KI @=@ Kiribati +dnet:countries @=@ dnet:countries @=@ KR @=@ Korea (Republic of) +dnet:countries @=@ dnet:countries @=@ KP @=@ Korea, Democatric People's Republic of +dnet:countries @=@ dnet:countries @=@ XK @=@ Kosovo * UN resolution +dnet:countries @=@ dnet:countries @=@ KW @=@ Kuwait +dnet:countries @=@ dnet:countries @=@ KG @=@ Kyrgyzstan +dnet:countries @=@ dnet:countries @=@ LA @=@ Lao (People's Democratic Republic) +dnet:countries @=@ dnet:countries @=@ LV @=@ Latvia +dnet:countries @=@ dnet:countries @=@ LB @=@ Lebanon +dnet:countries @=@ dnet:countries @=@ LS @=@ Lesotho +dnet:countries @=@ dnet:countries @=@ LR @=@ Liberia +dnet:countries @=@ dnet:countries @=@ LY @=@ Libyan Arab Jamahiriya +dnet:countries @=@ dnet:countries @=@ LI @=@ Liechtenstein +dnet:countries @=@ dnet:countries @=@ LT @=@ Lithuania +dnet:countries @=@ dnet:countries @=@ LU @=@ Luxembourg +dnet:countries @=@ dnet:countries @=@ MO @=@ Macao +dnet:countries @=@ dnet:countries @=@ MG @=@ Madagascar +dnet:countries @=@ dnet:countries @=@ MW @=@ Malawi +dnet:countries @=@ dnet:countries @=@ MY @=@ Malaysia +dnet:countries @=@ dnet:countries @=@ MV @=@ Maldives +dnet:countries @=@ dnet:countries @=@ ML @=@ Mali +dnet:countries @=@ dnet:countries @=@ MT @=@ Malta +dnet:countries @=@ dnet:countries @=@ MH @=@ Marshall Islands +dnet:countries @=@ dnet:countries @=@ MQ @=@ Martinique +dnet:countries @=@ dnet:countries @=@ MR @=@ Mauritania +dnet:countries @=@ dnet:countries @=@ MU @=@ Mauritius +dnet:countries @=@ dnet:countries @=@ YT @=@ Mayotte +dnet:countries @=@ dnet:countries @=@ MX @=@ Mexico +dnet:countries @=@ dnet:countries @=@ FM @=@ Micronesia, Federated States of +dnet:countries @=@ dnet:countries @=@ MD @=@ Moldova (Republic of) +dnet:countries @=@ dnet:countries @=@ MN @=@ Mongolia +dnet:countries @=@ dnet:countries @=@ ME @=@ Montenegro +dnet:countries @=@ dnet:countries @=@ MS @=@ Montserrat +dnet:countries @=@ dnet:countries @=@ MA @=@ Morocco +dnet:countries @=@ dnet:countries @=@ MZ @=@ Mozambique +dnet:countries @=@ dnet:countries @=@ MM @=@ Myanmar +dnet:countries @=@ dnet:countries @=@ NA @=@ Namibia +dnet:countries @=@ dnet:countries @=@ NR @=@ Nauru +dnet:countries @=@ dnet:countries @=@ NP @=@ Nepal +dnet:countries @=@ dnet:countries @=@ NL @=@ Netherlands +dnet:countries @=@ dnet:countries @=@ AN @=@ Netherlands Antilles +dnet:countries @=@ dnet:countries @=@ NC @=@ New Caledonia +dnet:countries @=@ dnet:countries @=@ NZ @=@ New Zealand +dnet:countries @=@ dnet:countries @=@ NI @=@ Nicaragua +dnet:countries @=@ dnet:countries @=@ NE @=@ Niger +dnet:countries @=@ dnet:countries @=@ NG @=@ Nigeria +dnet:countries @=@ dnet:countries @=@ NU @=@ Niue +dnet:countries @=@ dnet:countries @=@ NF @=@ Norfolk Island +dnet:countries @=@ dnet:countries @=@ MP @=@ Northern Mariana Islands +dnet:countries @=@ dnet:countries @=@ NO @=@ Norway +dnet:countries @=@ dnet:countries @=@ OC @=@ Oceania +dnet:countries @=@ dnet:countries @=@ OM @=@ Oman +dnet:countries @=@ dnet:countries @=@ PK @=@ Pakistan +dnet:countries @=@ dnet:countries @=@ PW @=@ Palau +dnet:countries @=@ dnet:countries @=@ PS @=@ Palestinian-administered areas +dnet:countries @=@ dnet:countries @=@ PA @=@ Panama +dnet:countries @=@ dnet:countries @=@ PG @=@ Papua New Guinea +dnet:countries @=@ dnet:countries @=@ PY @=@ Paraguay +dnet:countries @=@ dnet:countries @=@ PE @=@ Peru +dnet:countries @=@ dnet:countries @=@ PH @=@ Philippines +dnet:countries @=@ dnet:countries @=@ PN @=@ Pitcairn +dnet:countries @=@ dnet:countries @=@ PL @=@ Poland +dnet:countries @=@ dnet:countries @=@ PT @=@ Portugal +dnet:countries @=@ dnet:countries @=@ PR @=@ Puerto Rico +dnet:countries @=@ dnet:countries @=@ QA @=@ Qatar +dnet:countries @=@ dnet:countries @=@ RO @=@ Romania +dnet:countries @=@ dnet:countries @=@ RU @=@ Russian Federation +dnet:countries @=@ dnet:countries @=@ RW @=@ Rwanda +dnet:countries @=@ dnet:countries @=@ RE @=@ Réunion +dnet:countries @=@ dnet:countries @=@ SH @=@ Saint Helena, Ascension and Tristan da Cunha +dnet:countries @=@ dnet:countries @=@ KN @=@ Saint Kitts and Nevis +dnet:countries @=@ dnet:countries @=@ LC @=@ Saint Lucia +dnet:countries @=@ dnet:countries @=@ MF @=@ Saint Martin (French Part) +dnet:countries @=@ dnet:countries @=@ PM @=@ Saint Pierre and Miquelon +dnet:countries @=@ dnet:countries @=@ VC @=@ Saint Vincent and the Grenadines +dnet:countries @=@ dnet:countries @=@ BL @=@ Saint-Barthélemy +dnet:countries @=@ dnet:countries @=@ WS @=@ Samoa +dnet:countries @=@ dnet:countries @=@ SM @=@ San Marino +dnet:countries @=@ dnet:countries @=@ SA @=@ Saudi Arabia +dnet:countries @=@ dnet:countries @=@ SN @=@ Senegal +dnet:countries @=@ dnet:countries @=@ RS @=@ Serbia +dnet:countries @=@ dnet:countries @=@ CS @=@ Serbia and Montenegro +dnet:countries @=@ dnet:countries @=@ SC @=@ Seychelles +dnet:countries @=@ dnet:countries @=@ SL @=@ Sierra Leone +dnet:countries @=@ dnet:countries @=@ SG @=@ Singapore +dnet:countries @=@ dnet:countries @=@ SX @=@ Sint Maarten (Dutch Part) +dnet:countries @=@ dnet:countries @=@ SK @=@ Slovakia +dnet:countries @=@ dnet:countries @=@ SI @=@ Slovenia +dnet:countries @=@ dnet:countries @=@ SB @=@ Solomon Islands +dnet:countries @=@ dnet:countries @=@ SO @=@ Somalia +dnet:countries @=@ dnet:countries @=@ ZA @=@ South Africa +dnet:countries @=@ dnet:countries @=@ GS @=@ South Georgia and the South Sandwich Islands +dnet:countries @=@ dnet:countries @=@ SS @=@ South Sudan +dnet:countries @=@ dnet:countries @=@ ES @=@ Spain +dnet:countries @=@ dnet:countries @=@ LK @=@ Sri Lanka +dnet:countries @=@ dnet:countries @=@ SD @=@ Sudan +dnet:countries @=@ dnet:countries @=@ SR @=@ Suriname +dnet:countries @=@ dnet:countries @=@ SJ @=@ Svalbard and Jan Mayen +dnet:countries @=@ dnet:countries @=@ SZ @=@ Swaziland +dnet:countries @=@ dnet:countries @=@ SE @=@ Sweden +dnet:countries @=@ dnet:countries @=@ CH @=@ Switzerland +dnet:countries @=@ dnet:countries @=@ SY @=@ Syrian Arab Republic +dnet:countries @=@ dnet:countries @=@ ST @=@ São Tomé and Príncipe +dnet:countries @=@ dnet:countries @=@ TW @=@ Taiwan +dnet:countries @=@ dnet:countries @=@ TJ @=@ Tajikistan +dnet:countries @=@ dnet:countries @=@ TZ @=@ Tanzania (United Republic of) +dnet:countries @=@ dnet:countries @=@ TH @=@ Thailand +dnet:countries @=@ dnet:countries @=@ TL @=@ Timor-Leste +dnet:countries @=@ dnet:countries @=@ TG @=@ Togo +dnet:countries @=@ dnet:countries @=@ TK @=@ Tokelau +dnet:countries @=@ dnet:countries @=@ TO @=@ Tonga +dnet:countries @=@ dnet:countries @=@ TT @=@ Trinidad and Tobago +dnet:countries @=@ dnet:countries @=@ TN @=@ Tunisia +dnet:countries @=@ dnet:countries @=@ TR @=@ Turkey +dnet:countries @=@ dnet:countries @=@ TM @=@ Turkmenistan +dnet:countries @=@ dnet:countries @=@ TC @=@ Turks and Caicos Islands +dnet:countries @=@ dnet:countries @=@ TV @=@ Tuvalu +dnet:countries @=@ dnet:countries @=@ UNKNOWN @=@ UNKNOWN +dnet:countries @=@ dnet:countries @=@ UG @=@ Uganda +dnet:countries @=@ dnet:countries @=@ UA @=@ Ukraine +dnet:countries @=@ dnet:countries @=@ AE @=@ United Arab Emirates +dnet:countries @=@ dnet:countries @=@ GB @=@ United Kingdom +dnet:countries @=@ dnet:countries @=@ US @=@ United States +dnet:countries @=@ dnet:countries @=@ UM @=@ United States Minor Outlying Islands +dnet:countries @=@ dnet:countries @=@ UY @=@ Uruguay +dnet:countries @=@ dnet:countries @=@ UZ @=@ Uzbekistan +dnet:countries @=@ dnet:countries @=@ VU @=@ Vanuatu +dnet:countries @=@ dnet:countries @=@ VE @=@ Venezuela +dnet:countries @=@ dnet:countries @=@ VN @=@ Viet Nam +dnet:countries @=@ dnet:countries @=@ VG @=@ Virgin Islands (British) +dnet:countries @=@ dnet:countries @=@ VI @=@ Virgin Islands, U.S. +dnet:countries @=@ dnet:countries @=@ WF @=@ Wallis and Futuna +dnet:countries @=@ dnet:countries @=@ EH @=@ Western Sahara +dnet:countries @=@ dnet:countries @=@ YE @=@ Yemen +dnet:countries @=@ dnet:countries @=@ YU @=@ Yugoslavia +dnet:countries @=@ dnet:countries @=@ ZM @=@ Zambia +dnet:countries @=@ dnet:countries @=@ ZW @=@ Zimbabwe +dnet:countries @=@ dnet:countries @=@ AX @=@ Åland Islands +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire2.0 @=@ OpenAIRE 2.0 (EC funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ driver-openaire2.0 @=@ OpenAIRE 2.0+ (DRIVER OA, EC funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire3.0 @=@ OpenAIRE 3.0 (OA, funding) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire4.0 @=@ OpenAIRE 4.0 (inst.&thematic. repo.) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ driver @=@ OpenAIRE Basic (DRIVER OA) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire2.0_data @=@ OpenAIRE Data (funded, referenced datasets) +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ hostedBy @=@ collected from a compatible aggregator +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ UNKNOWN @=@ not available +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ native @=@ proprietary +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ notCompatible @=@ under validation +dnet:datasourceCompatibilityLevel @=@ dnet:datasourceCompatibilityLevel @=@ openaire-cris_1.1 @=@ OpenAIRE CRIS v1.1 +fct:funding_relations @=@ fct:funding_relations @=@ fct:hasParentFunding @=@ fct:hasParentFunding dnet:protocols @=@ dnet:protocols @=@ HTTPWithFileName @=@ HTTPWithFileName dnet:protocols @=@ dnet:protocols @=@ NetCDF @=@ NetCDF dnet:protocols @=@ dnet:protocols @=@ OpenDAP @=@ OpenDAP dnet:protocols @=@ dnet:protocols @=@ schemaorg @=@ Schema.org dnet:protocols @=@ dnet:protocols @=@ UNKNOWN @=@ UNKNOWN dnet:protocols @=@ dnet:protocols @=@ api @=@ api -dnet:protocols @=@ dnet:protocols @=@ dataciteESPlugin @=@ dataciteESPlugin +dnet:protocols @=@ dnet:protocols @=@ dataciteESPlugins @=@ dataciteESPlugins dnet:protocols @=@ dnet:protocols @=@ datasetsbyjournal @=@ datasetsbyjournal dnet:protocols @=@ dnet:protocols @=@ datasetsbyproject @=@ datasetsbyproject dnet:protocols @=@ dnet:protocols @=@ excelFile @=@ excelFile dnet:protocols @=@ dnet:protocols @=@ file @=@ file -dnet:protocols @=@ dnet:protocols @=@ fileCSV @=@ fileCSV dnet:protocols @=@ dnet:protocols @=@ fileGzip @=@ fileGzip dnet:protocols @=@ dnet:protocols @=@ files_by_rpc @=@ files_by_rpc dnet:protocols @=@ dnet:protocols @=@ files_from_mdstore @=@ files_from_mdstore @@ -917,128 +999,81 @@ dnet:protocols @=@ dnet:protocols @=@ sparql @=@ sparql dnet:protocols @=@ dnet:protocols @=@ sword @=@ sword dnet:protocols @=@ dnet:protocols @=@ targz @=@ targz dnet:protocols @=@ dnet:protocols @=@ remoteMdstore @=@ remoteMdstore -ec:funding_typologies @=@ ec:funding_typologies @=@ ec:frameworkprogram @=@ frameworkprogram -ec:funding_typologies @=@ ec:funding_typologies @=@ ec:program @=@ program -ec:funding_typologies @=@ ec:funding_typologies @=@ ec:specificprogram @=@ specificprogram -fct:contractTypes @=@ fct:contractTypes @=@ UNKNOWN @=@ UNKNOWN -wt:contractTypes @=@ wt:contractTypes @=@ UNKNOWN @=@ UNKNOWN -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ UNKNOWN @=@ UNKNOWN -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ collection @=@ collection -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ dataset @=@ dataset -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ event @=@ event -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ film @=@ film -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ image @=@ image -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ interactiveResource @=@ interactiveResource -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ model @=@ model -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ physicalObject @=@ physicalObject -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ service @=@ service -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ software @=@ software -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ sound @=@ sound -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ text @=@ text -dnet:dataCite_resource @=@ dnet:dataCite_resource @=@ clinicalTrial @=@ clinical trial -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire2.0 @=@ OpenAIRE 2.0 (EC funding) -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire3.0 @=@ OpenAIRE 3.0 (OA, funding) -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire4.0 @=@ OpenAIRE 4.0 (inst.&thematic. repo.) -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ driver @=@ OpenAIRE Basic (DRIVER OA) -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire2.0_data @=@ OpenAIRE Data (funded, referenced datasets) -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ hostedBy @=@ collected from a compatible aggregator -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ files @=@ files -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ native @=@ native -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ UNKNOWN @=@ not available -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ notCompatible @=@ under validation -dnet:compatibilityLevel @=@ dnet:compatibilityLevel @=@ openaire-cris_1.1 @=@ OpenAIRE CRIS v1.1 -dnet:publication_resource @=@ dnet:publication_resource @=@ 0018 @=@ Annotation -dnet:publication_resource @=@ dnet:publication_resource @=@ 0001 @=@ Article -dnet:publication_resource @=@ dnet:publication_resource @=@ 0033 @=@ Audiovisual -dnet:publication_resource @=@ dnet:publication_resource @=@ 0008 @=@ Bachelor thesis -dnet:publication_resource @=@ dnet:publication_resource @=@ 0002 @=@ Book -dnet:publication_resource @=@ dnet:publication_resource @=@ 0037 @=@ Clinical Trial -dnet:publication_resource @=@ dnet:publication_resource @=@ 0022 @=@ Collection -dnet:publication_resource @=@ dnet:publication_resource @=@ 0004 @=@ Conference object -dnet:publication_resource @=@ dnet:publication_resource @=@ 0005 @=@ Contribution for newspaper or weekly magazine -dnet:publication_resource @=@ dnet:publication_resource @=@ 0045 @=@ Data Management Plan -dnet:publication_resource @=@ dnet:publication_resource @=@ 0031 @=@ Data Paper -dnet:publication_resource @=@ dnet:publication_resource @=@ 0021 @=@ Dataset -dnet:publication_resource @=@ dnet:publication_resource @=@ 0006 @=@ Doctoral thesis -dnet:publication_resource @=@ dnet:publication_resource @=@ 0023 @=@ Event -dnet:publication_resource @=@ dnet:publication_resource @=@ 0009 @=@ External research report -dnet:publication_resource @=@ dnet:publication_resource @=@ 0024 @=@ Film -dnet:publication_resource @=@ dnet:publication_resource @=@ 0025 @=@ Image -dnet:publication_resource @=@ dnet:publication_resource @=@ 0026 @=@ InteractiveResource -dnet:publication_resource @=@ dnet:publication_resource @=@ 0011 @=@ Internal report -dnet:publication_resource @=@ dnet:publication_resource @=@ 0043 @=@ Journal -dnet:publication_resource @=@ dnet:publication_resource @=@ 0010 @=@ Lecture -dnet:publication_resource @=@ dnet:publication_resource @=@ 0007 @=@ Master thesis -dnet:publication_resource @=@ dnet:publication_resource @=@ 0027 @=@ Model -dnet:publication_resource @=@ dnet:publication_resource @=@ 0012 @=@ Newsletter -dnet:publication_resource @=@ dnet:publication_resource @=@ 0020 @=@ Other ORP type -dnet:publication_resource @=@ dnet:publication_resource @=@ 0039 @=@ Other dataset type -dnet:publication_resource @=@ dnet:publication_resource @=@ 0038 @=@ Other literature type -dnet:publication_resource @=@ dnet:publication_resource @=@ 0040 @=@ Other software type -dnet:publication_resource @=@ dnet:publication_resource @=@ 0013 @=@ Part of book or chapter of book -dnet:publication_resource @=@ dnet:publication_resource @=@ 0019 @=@ Patent -dnet:publication_resource @=@ dnet:publication_resource @=@ 0028 @=@ PhysicalObject -dnet:publication_resource @=@ dnet:publication_resource @=@ 0016 @=@ Preprint -dnet:publication_resource @=@ dnet:publication_resource @=@ 0034 @=@ Project deliverable -dnet:publication_resource @=@ dnet:publication_resource @=@ 0035 @=@ Project milestone -dnet:publication_resource @=@ dnet:publication_resource @=@ 0036 @=@ Project proposal -dnet:publication_resource @=@ dnet:publication_resource @=@ 0017 @=@ Report -dnet:publication_resource @=@ dnet:publication_resource @=@ 0014 @=@ Research -dnet:publication_resource @=@ dnet:publication_resource @=@ 0015 @=@ Review -dnet:publication_resource @=@ dnet:publication_resource @=@ 0029 @=@ Software -dnet:publication_resource @=@ dnet:publication_resource @=@ 0032 @=@ Software Paper -dnet:publication_resource @=@ dnet:publication_resource @=@ 0030 @=@ Sound -dnet:publication_resource @=@ dnet:publication_resource @=@ 0044 @=@ Thesis -dnet:publication_resource @=@ dnet:publication_resource @=@ 0000 @=@ UNKNOWN -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:aggregator @=@ Harvested -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:cris @=@ Harvested -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:datasetarchive @=@ Harvested -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:entityregistry @=@ Harvested -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:infospace @=@ Harvested -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk:repository @=@ Harvested -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:aggregator @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:cris @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:datasetarchive @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:entityregistry @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:infospace @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:mining:repository @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ UNKNOWN @=@ UNKNOWN -dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim:pid @=@ Linked by user -dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim:search @=@ Linked by user -dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:insert @=@ Linked by user -dnet:provenanceActions @=@ dnet:provenanceActions @=@ user:claim @=@ Linked by user -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:actionset @=@ Harvested -dnet:provenanceActions @=@ dnet:provenanceActions @=@ iis @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ propagation:project:semrel @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:actionset:orcidworks-no-doi @=@ Harvested -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:dedup @=@ Inferred by OpenAIRE -dnet:provenanceActions @=@ dnet:provenanceActions @=@ sysimport:crosswalk @=@ Harvested -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ crissystem @=@ CRIS System -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ datarepository::unknown @=@ Data Repository -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::datarepository @=@ Data Repository Aggregator -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ infospace @=@ Information Space -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::institutional @=@ Institutional Repository -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::institutional @=@ Institutional Repository Aggregator -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::journal @=@ Journal -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::journals @=@ Journal Aggregator/Publisher -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::mock @=@ Other -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubscatalogue::unknown @=@ Publication Catalogue -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::unknown @=@ Publication Repository -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::pubsrepository::unknown @=@ Publication Repository Aggregator -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry @=@ Registry -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ scholarcomminfra @=@ Scholarly Comm. Infrastructure -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ pubsrepository::thematic @=@ Thematic Repository -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ websource @=@ Web Source -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::projects @=@ Funder database -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::repositories @=@ Registry of repositories -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ softwarerepository @=@ Software Repository -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ aggregator::softwarerepository @=@ Software Repository Aggregator -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ orprepository @=@ Repository -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ researchgraph @=@ Research Graph -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::products @=@ Registry of research products -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::researchers @=@ Registry of researchers -dnet:datasource_typologies @=@ dnet:datasource_typologies @=@ entityregistry::organizations @=@ Registry of organizations wt:funding_typologies @=@ Wellcome Trust: Funding Typologies @=@ wt:fundingStream @=@ Wellcome Trust: Funding Stream -dnet:review_levels @=@ dnet:review_levels @=@ 0000 @=@ UNKNOWN +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ accessionNumber @=@ accessionNumber +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ dataset @=@ dataset +dnet:externalReference_typologies @=@ dnet:externalReference_typologies @=@ software @=@ software +datacite:id_typologies @=@ datacite:id_typologies @=@ ARK @=@ ARK +datacite:id_typologies @=@ datacite:id_typologies @=@ DOI @=@ DOI +datacite:id_typologies @=@ datacite:id_typologies @=@ EAN13 @=@ EAN13 +datacite:id_typologies @=@ datacite:id_typologies @=@ EISSN @=@ EISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ Handle @=@ Handle +datacite:id_typologies @=@ datacite:id_typologies @=@ ISBN @=@ ISBN +datacite:id_typologies @=@ datacite:id_typologies @=@ ISSN @=@ ISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ ISTC @=@ ISTC +datacite:id_typologies @=@ datacite:id_typologies @=@ LISSN @=@ LISSN +datacite:id_typologies @=@ datacite:id_typologies @=@ LSID @=@ LSID +datacite:id_typologies @=@ datacite:id_typologies @=@ PURL @=@ PURL +datacite:id_typologies @=@ datacite:id_typologies @=@ UNKNOWN @=@ UNKNOWN +datacite:id_typologies @=@ datacite:id_typologies @=@ UPC @=@ UPC +datacite:id_typologies @=@ datacite:id_typologies @=@ URL @=@ URL +datacite:id_typologies @=@ datacite:id_typologies @=@ URN @=@ URN +dnet:pid_types @=@ dnet:pid_types @=@ actrn @=@ ACTRN Identifier +dnet:pid_types @=@ dnet:pid_types @=@ nct @=@ ClinicalTrials.gov Identifier +dnet:pid_types @=@ dnet:pid_types @=@ euctr @=@ EU Clinical Trials Register +dnet:pid_types @=@ dnet:pid_types @=@ epo_id @=@ European Patent Office application ID +dnet:pid_types @=@ dnet:pid_types @=@ gsk @=@ GSK Identifier +dnet:pid_types @=@ dnet:pid_types @=@ GeoPass @=@ Geographic Location-Password Scheme +dnet:pid_types @=@ dnet:pid_types @=@ GBIF @=@ Global Biodiversity Information Facility +dnet:pid_types @=@ dnet:pid_types @=@ isrctn @=@ ISRCTN Identifier +dnet:pid_types @=@ dnet:pid_types @=@ ISNI @=@ International Standard Name Identifier +dnet:pid_types @=@ dnet:pid_types @=@ jprn @=@ JPRN Identifier +dnet:pid_types @=@ dnet:pid_types @=@ mag_id @=@ Microsoft Academic Graph Identifier +dnet:pid_types @=@ dnet:pid_types @=@ oai @=@ Open Archives Initiative +dnet:pid_types @=@ dnet:pid_types @=@ orcid @=@ Open Researcher and Contributor ID +dnet:pid_types @=@ dnet:pid_types @=@ PANGAEA @=@ PANGAEA +dnet:pid_types @=@ dnet:pid_types @=@ epo_nr_epodoc @=@ Patent application number in EPODOC format +dnet:pid_types @=@ dnet:pid_types @=@ UNKNOWN @=@ UNKNOWN +dnet:pid_types @=@ dnet:pid_types @=@ VIAF @=@ Virtual International Authority File +dnet:pid_types @=@ dnet:pid_types @=@ arXiv @=@ arXiv +dnet:pid_types @=@ dnet:pid_types @=@ doi @=@ doi +dnet:pid_types @=@ dnet:pid_types @=@ grid @=@ grid +dnet:pid_types @=@ dnet:pid_types @=@ info:eu-repo/dai @=@ info:eu-repo/dai +dnet:pid_types @=@ dnet:pid_types @=@ orcidworkid @=@ orcid workid +dnet:pid_types @=@ dnet:pid_types @=@ pmc @=@ pmc +dnet:pid_types @=@ dnet:pid_types @=@ pmid @=@ pmid +dnet:pid_types @=@ dnet:pid_types @=@ urn @=@ urn +dnet:pid_types @=@ dnet:pid_types @=@ who @=@ WHO Identifier +dnet:pid_types @=@ dnet:pid_types @=@ drks @=@ DRKS Identifier +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/ACM @=@ An ACM classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/ARXIV @=@ An ARXIV classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/DDC @=@ A Dewey Decimal classification term (DDC) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/JEL @=@ A Journal of Economic Literature (JEL) classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/OPENACCESS_VERSION @=@ An Open Access versions of your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/IS_REFERENCED_BY @=@ A dataset referenced by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/REFERENCES @=@ A dataset that refers to your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/IS_RELATED_TO @=@ A dataset related to your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/IS_SUPPLEMENTED_TO @=@ A dataset that supplements your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/IS_RELATED_TO @=@ A publication related to your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/REFERENCES @=@ A publication referenced by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/IS_REFERENCED_BY @=@ A publication that refers to your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/IS_SUPPLEMENTED_BY @=@ A publication that is supplemented by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION/IS_SUPPLEMENTED_TO @=@ A publication that supplements your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SOFTWARE @=@ A software referred by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/OPENACCESS_VERSION @=@ Another Open Access version of a publication +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/PID @=@ Another persistent identifier associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/SUBJECT/MESHEUROPMC @=@ A classification term from the Medical Subject Headings (MeSH) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/ABSTRACT @=@ An abstract describing among your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PUBLICATION_DATE @=@ A date of publication missing in your content +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PID @=@ A persistent identifier associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/ACM @=@ Another ACM classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/ARXIV @=@ Another ARXIV classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/DDC @=@ Another Dewey Decimal classification term (DDC) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/JEL @=@ Another Journal of Economic Literature (JEL) classification term that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MORE/SUBJECT/MESHEUROPMC @=@ Another classification term from the Medical Subject Headings (MeSH) that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/PROJECT @=@ A project reference that can be associated to your publications +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/DATASET/IS_SUPPLEMENTED_BY @=@ A dataset that is supplemented by your records +dnet:topic_types @=@ dnet:topic_types @=@ ENRICH/MISSING/AUTHOR/ORCID @=@ An Open Researcher and Contributor ID (ORCID) that can be associated to an author of your publications +dnet:review_levels @=@ dnet:review_levels @=@ 0000 @=@ Unknown dnet:review_levels @=@ dnet:review_levels @=@ 0002 @=@ nonPeerReviewed dnet:review_levels @=@ dnet:review_levels @=@ 0001 @=@ peerReviewed \ No newline at end of file From 8a4f84f8c03cc38bba6f928b631b0d2e85ffd43a Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 16 Jun 2020 12:34:13 +0200 Subject: [PATCH 81/88] refactoring --- dhp-workflows/dhp-broker-events/pom.xml | 2 +- dhp-workflows/dhp-broker-events/report.xml | 37 ++++ .../eu/dnetlib/dhp/broker/model/Event.java | 8 +- .../dhp/broker/model/EventFactory.java | 53 ++--- .../broker/oa/GenerateEventsApplication.java | 54 +++-- .../dhp/broker/oa/matchers/UpdateMatcher.java | 29 ++- .../AbstractEnrichMissingDataset.java | 22 +- .../relatedProjects/EnrichMissingProject.java | 23 +- .../relatedProjects/EnrichMoreProject.java | 29 ++- .../AbstractEnrichMissingPublication.java | 22 +- .../EnrichMissingSoftware.java | 22 +- .../relatedSoftware/EnrichMoreSoftware.java | 20 +- .../simple/EnrichMissingAbstract.java | 13 +- .../simple/EnrichMissingAuthorOrcid.java | 44 ++-- .../simple/EnrichMissingOpenAccess.java | 19 +- .../oa/matchers/simple/EnrichMissingPid.java | 17 +- .../simple/EnrichMissingPublicationDate.java | 15 +- .../matchers/simple/EnrichMissingSubject.java | 38 ++-- .../matchers/simple/EnrichMoreOpenAccess.java | 20 +- .../oa/matchers/simple/EnrichMorePid.java | 27 ++- .../oa/matchers/simple/EnrichMoreSubject.java | 32 ++- .../dhp/broker/oa/util/BrokerConstants.java | 19 +- .../dhp/broker/oa/util/ConversionUtils.java | 51 ++++- .../dhp/broker/oa/util/EventFinder.java | 4 +- .../dhp/broker/oa/util/UpdateInfo.java | 51 ++--- .../aggregators/simple/ResultAggregator.java | 6 +- .../util/aggregators/simple/ResultGroup.java | 8 +- .../OpenaireBrokerResultAggregator.java | 69 ++++++ .../aggregators/withRels/RelatedDataset.java | 10 +- .../withRels/RelatedEntityFactory.java | 12 +- .../aggregators/withRels/RelatedProject.java | 11 +- .../withRels/RelatedPublication.java | 11 +- .../aggregators/withRels/RelatedSoftware.java | 10 +- .../withRels/ResultWithRelations.java | 55 ----- .../ResultWithRelationsAggregator.java | 68 ------ .../oa/graph/reflections/ReflectionTest.java | 200 ++++++++++++++++++ 36 files changed, 640 insertions(+), 491 deletions(-) create mode 100644 dhp-workflows/dhp-broker-events/report.xml create mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/OpenaireBrokerResultAggregator.java delete mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelations.java delete mode 100644 dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelationsAggregator.java create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/reflections/ReflectionTest.java diff --git a/dhp-workflows/dhp-broker-events/pom.xml b/dhp-workflows/dhp-broker-events/pom.xml index f060a60bc..eddc042c6 100644 --- a/dhp-workflows/dhp-broker-events/pom.xml +++ b/dhp-workflows/dhp-broker-events/pom.xml @@ -61,7 +61,7 @@ eu.dnetlib dnet-openaire-broker-common - [2.0.1,3.0.0) + [3.0.1,4.0.0) diff --git a/dhp-workflows/dhp-broker-events/report.xml b/dhp-workflows/dhp-broker-events/report.xml new file mode 100644 index 000000000..6e706f723 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/report.xml @@ -0,0 +1,37 @@ + + + + Feature Extraction + + + TCPFLOW + 1.5.0 + + 4.2.1 (4.2.1 Compatible Apple LLVM 11.0.0 (clang-1100.0.33.8)) + -D_THREAD_SAFE -pthread -I/usr/local/include -I/usr/local/include -DUTC_OFFSET=+0000 + -g -D_THREAD_SAFE -pthread -g -O3 -MD -Wpointer-arith -Wmissing-declarations -Wmissing-prototypes -Wshadow -Wwrite-strings -Wcast-align -Waggregate-return -Wbad-function-cast -Wcast-qual -Wundef -Wredundant-decls -Wdisabled-optimization -Wfloat-equal -Wmultichar -Wc++-compat -Wmissing-noreturn -Wall -Wstrict-prototypes -MD -D_FORTIFY_SOURCE=2 -Wpointer-arith -Wmissing-declarations -Wmissing-prototypes -Wshadow -Wwrite-strings -Wcast-align -Waggregate-return -Wbad-function-cast -Wcast-qual -Wundef -Wredundant-decls -Wdisabled-optimization -Wfloat-equal -Wmultichar -Wc++-compat -Wmissing-noreturn -Wall -Wstrict-prototypes + -g -D_THREAD_SAFE -pthread -g -O3 -Wall -MD -D_FORTIFY_SOURCE=2 -Wpointer-arith -Wshadow -Wwrite-strings -Wcast-align -Wredundant-decls -Wdisabled-optimization -Wfloat-equal -Wmultichar -Wmissing-noreturn -Woverloaded-virtual -Wsign-promo -funit-at-a-time -Weffc++ -std=c++11 -Wall -MD -D_FORTIFY_SOURCE=2 -Wpointer-arith -Wshadow -Wwrite-strings -Wcast-align -Wredundant-decls -Wdisabled-optimization -Wfloat-equal -Wmultichar -Wmissing-noreturn -Woverloaded-virtual -Wsign-promo -funit-at-a-time -Weffc++ + -L/usr/local/lib -L/usr/local/lib + -lpython2.7 -lpython2.7 -lpcap -lbz2 -lexpat -lsqlite3 -lcrypto -lssl -lcrypto -ldl -lz + 2019-10-11T01:16:58 + + + + + Darwin + 19.5.0 + Darwin Kernel Version 19.5.0: Tue May 26 20:41:44 PDT 2020; root:xnu-6153.121.2~2/RELEASE_X86_64 + Micheles-MBP.local + x86_64 + tcpflow + 501 + michele + 2020-06-15T14:55:03Z + + + + + 0 diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/Event.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/Event.java index 0512a3813..f94d286e4 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/Event.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/Event.java @@ -1,9 +1,15 @@ package eu.dnetlib.dhp.broker.model; +import java.io.Serializable; import java.util.Map; -public class Event { +public class Event implements Serializable { + + /** + * + */ + private static final long serialVersionUID = -5936790326505781395L; private String eventId; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java index 9146cf422..bf4f62d24 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/model/EventFactory.java @@ -6,17 +6,13 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.schema.oaf.Author; -import eu.dnetlib.dhp.schema.oaf.KeyValue; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.StructuredProperty; public class EventFactory { @@ -37,8 +33,7 @@ public class EventFactory { final Map map = createMapFromResult(updateInfo); final String eventId = calculateEventId( - updateInfo.getTopicPath(), updateInfo.getTarget().getResult().getOriginalId().get(0), - updateInfo.getHighlightValueAsString()); + updateInfo.getTopicPath(), updateInfo.getTarget().getOriginalId(), updateInfo.getHighlightValueAsString()); res.setEventId(eventId); res.setProducerId(PRODUCER_ID); @@ -54,53 +49,31 @@ public class EventFactory { private static Map createMapFromResult(final UpdateInfo updateInfo) { final Map map = new HashMap<>(); - final Result source = updateInfo.getSource().getResult(); - final Result target = updateInfo.getTarget().getResult(); + final OpenaireBrokerResult source = updateInfo.getSource(); + final OpenaireBrokerResult target = updateInfo.getTarget(); - final List collectedFrom = target.getCollectedfrom(); - if (collectedFrom.size() == 1) { - map.put("target_datasource_id", collectedFrom.get(0).getKey()); - map.put("target_datasource_name", collectedFrom.get(0).getValue()); - } + map.put("target_datasource_id", target.getCollectedFromId()); + map.put("target_datasource_name", target.getCollectedFromName()); - final List ids = target.getOriginalId(); - if (ids.size() > 0) { - map.put("target_publication_id", ids.get(0)); - } + map.put("target_publication_id", target.getOriginalId()); - final List titles = target.getTitle(); + final List titles = target.getTitles(); if (titles.size() > 0) { map.put("target_publication_title", titles.get(0)); } - final long date = parseDateTolong(target.getDateofacceptance().getValue()); + final long date = parseDateTolong(target.getPublicationdate()); if (date > 0) { map.put("target_dateofacceptance", date); } - final List subjects = target.getSubject(); - if (subjects.size() > 0) { - map - .put( - "target_publication_subject_list", - subjects.stream().map(StructuredProperty::getValue).collect(Collectors.toList())); - } - - final List authors = target.getAuthor(); - if (authors.size() > 0) { - map - .put( - "target_publication_author_list", - authors.stream().map(Author::getFullname).collect(Collectors.toList())); - } + map.put("target_publication_subject_list", target.getSubjects()); + map.put("target_publication_author_list", target.getCreators()); // PROVENANCE INFO map.put("trust", updateInfo.getTrust()); - final List sourceCollectedFrom = source.getCollectedfrom(); - if (sourceCollectedFrom.size() == 1) { - map.put("provenance_datasource_id", sourceCollectedFrom.get(0).getKey()); - map.put("provenance_datasource_name", sourceCollectedFrom.get(0).getValue()); - } + map.put("provenance_datasource_id", source.getCollectedFromId()); + map.put("provenance_datasource_name", source.getCollectedFromName()); map.put("provenance_publication_id_list", source.getOriginalId()); return map; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java index 4d40ba80d..940d7f9f3 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java @@ -18,18 +18,20 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.broker.model.Event; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; +import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; import eu.dnetlib.dhp.broker.oa.util.EventFinder; import eu.dnetlib.dhp.broker.oa.util.EventGroup; import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultAggregator; import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.OpenaireBrokerResultAggregator; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedEntityFactory; import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelationsAggregator; import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct; import eu.dnetlib.dhp.schema.oaf.Project; import eu.dnetlib.dhp.schema.oaf.Publication; import eu.dnetlib.dhp.schema.oaf.Relation; @@ -73,6 +75,8 @@ public class GenerateEventsApplication { log.info("dedupConfigProfileId: {}", dedupConfigProfileId); final SparkConf conf = new SparkConf(); + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + conf.registerKryoClasses(BrokerConstants.getModelClasses()); final DedupConfig dedupConfig = loadDedupConfig(isLookupUrl, dedupConfigProfileId); @@ -80,13 +84,16 @@ public class GenerateEventsApplication { removeOutputDir(spark, eventsPath); - final Dataset all = spark.emptyDataset(Encoders.kryo(Event.class)); - - for (final Class r1 : BrokerConstants.RESULT_CLASSES) { - all.union(generateEvents(spark, graphPath, r1, dedupConfig)); - } - - all.write().mode(SaveMode.Overwrite).option("compression", "gzip").json(eventsPath); + spark + .emptyDataset(Encoders.kryo(Event.class)) + .union(generateEvents(spark, graphPath, Publication.class, dedupConfig)) + .union(generateEvents(spark, graphPath, eu.dnetlib.dhp.schema.oaf.Dataset.class, dedupConfig)) + .union(generateEvents(spark, graphPath, Software.class, dedupConfig)) + .union(generateEvents(spark, graphPath, OtherResearchProduct.class, dedupConfig)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(eventsPath); }); } @@ -101,18 +108,18 @@ public class GenerateEventsApplication { final Class sourceClass, final DedupConfig dedupConfig) { - final Dataset results = expandResultsWithRelations(spark, graphPath, sourceClass); + final Dataset results = expandResultsWithRelations(spark, graphPath, sourceClass); final Dataset mergedRels = readPath(spark, graphPath + "/relation", Relation.class) .filter(r -> r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); - final TypedColumn, ResultGroup> aggr = new ResultAggregator() + final TypedColumn, ResultGroup> aggr = new ResultAggregator() .toColumn(); return results .joinWith(mergedRels, results.col("result.id").equalTo(mergedRels.col("source")), "inner") .groupByKey( - (MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) + (MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) .agg(aggr) .map((MapFunction, ResultGroup>) t -> t._2, Encoders.kryo(ResultGroup.class)) .filter(ResultGroup::isValid) @@ -122,7 +129,7 @@ public class GenerateEventsApplication { .flatMap(group -> group.getData().iterator(), Encoders.kryo(Event.class)); } - private static Dataset expandResultsWithRelations( + private static Dataset expandResultsWithRelations( final SparkSession spark, final String graphPath, final Class sourceClass) { @@ -135,14 +142,15 @@ public class GenerateEventsApplication { final Dataset rels = readPath(spark, graphPath + "/relation", Relation.class) .filter(r -> !r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); - final Dataset r0 = readPath( + final Dataset r0 = readPath( spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) .filter(r -> r.getDataInfo().getDeletedbyinference()) - .map(r -> new ResultWithRelations(r), Encoders.kryo(ResultWithRelations.class)); - final Dataset r1 = join(r0, rels, relatedEntities(projects, rels, RelatedProject.class)); - final Dataset r2 = join(r1, rels, relatedEntities(softwares, rels, RelatedProject.class)); - final Dataset r3 = join(r2, rels, relatedEntities(datasets, rels, RelatedProject.class)); - final Dataset r4 = join( + .map(ConversionUtils::oafResultToBrokerResult, Encoders.kryo(OpenaireBrokerResult.class)); + + final Dataset r1 = join(r0, rels, relatedEntities(projects, rels, RelatedProject.class)); + final Dataset r2 = join(r1, rels, relatedEntities(softwares, rels, RelatedProject.class)); + final Dataset r3 = join(r2, rels, relatedEntities(datasets, rels, RelatedProject.class)); + final Dataset r4 = join( r3, rels, relatedEntities(publications, rels, RelatedProject.class)); ; @@ -159,20 +167,20 @@ public class GenerateEventsApplication { Encoders.kryo(clazz)); } - private static Dataset join(final Dataset sources, + private static Dataset join(final Dataset sources, final Dataset rels, final Dataset typedRels) { - final TypedColumn, ResultWithRelations> aggr = new ResultWithRelationsAggregator() + final TypedColumn, OpenaireBrokerResult> aggr = new OpenaireBrokerResultAggregator() .toColumn(); ; return sources .joinWith(typedRels, sources.col("result.id").equalTo(rels.col("source")), "left_outer") .groupByKey( - (MapFunction, String>) t -> t._1.getResult().getId(), Encoders.STRING()) + (MapFunction, String>) t -> t._1.getOpenaireId(), Encoders.STRING()) .agg(aggr) - .map(t -> t._2, Encoders.kryo(ResultWithRelations.class)); + .map(t -> t._2, Encoders.kryo(OpenaireBrokerResult.class)); } public static Dataset readPath( diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java index 0e57f32f9..13aeefb2f 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/UpdateMatcher.java @@ -12,22 +12,20 @@ import java.util.function.Function; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; -import eu.dnetlib.broker.objects.Publication; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.util.UpdateInfo; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Field; import eu.dnetlib.pace.config.DedupConfig; public abstract class UpdateMatcher { private final boolean multipleUpdate; private final Function topicFunction; - private final BiConsumer compileHighlightFunction; + private final BiConsumer compileHighlightFunction; private final Function highlightToStringFunction; public UpdateMatcher(final boolean multipleUpdate, final Function topicFunction, - final BiConsumer compileHighlightFunction, + final BiConsumer compileHighlightFunction, final Function highlightToStringFunction) { this.multipleUpdate = multipleUpdate; this.topicFunction = topicFunction; @@ -35,19 +33,18 @@ public abstract class UpdateMatcher { this.highlightToStringFunction = highlightToStringFunction; } - public Collection> searchUpdatesForRecord(final ResultWithRelations res, - final Collection others, + public Collection> searchUpdatesForRecord(final OpenaireBrokerResult res, + final Collection others, final DedupConfig dedupConfig) { final Map> infoMap = new HashMap<>(); - for (final ResultWithRelations source : others) { + for (final OpenaireBrokerResult source : others) { if (source != res) { for (final T hl : findDifferences(source, res)) { final Topic topic = getTopicFunction().apply(hl); final UpdateInfo info = new UpdateInfo<>(topic, hl, source, res, getCompileHighlightFunction(), - getHighlightToStringFunction(), - dedupConfig); + getHighlightToStringFunction(), dedupConfig); final String s = DigestUtils.md5Hex(info.getHighlightValueAsString()); if (!infoMap.containsKey(s) || infoMap.get(s).getTrust() < info.getTrust()) { } else { @@ -71,14 +68,14 @@ public abstract class UpdateMatcher { } } - protected abstract List findDifferences(ResultWithRelations source, ResultWithRelations target); + protected abstract List findDifferences(OpenaireBrokerResult source, OpenaireBrokerResult target); - protected static boolean isMissing(final List> list) { - return list == null || list.isEmpty() || StringUtils.isBlank(list.get(0).getValue()); + protected static boolean isMissing(final List list) { + return list == null || list.isEmpty() || StringUtils.isBlank(list.get(0)); } - protected boolean isMissing(final Field field) { - return field == null || StringUtils.isBlank(field.getValue()); + protected boolean isMissing(final String field) { + return StringUtils.isBlank(field); } public boolean isMultipleUpdate() { @@ -89,7 +86,7 @@ public abstract class UpdateMatcher { return topicFunction; } - public BiConsumer getCompileHighlightFunction() { + public BiConsumer getCompileHighlightFunction() { return compileHighlightFunction; } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java index a8e0a2c42..7a58f986b 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedDatasets/AbstractEnrichMissingDataset.java @@ -5,45 +5,39 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import eu.dnetlib.broker.objects.Dataset; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedDataset; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Dataset; public abstract class AbstractEnrichMissingDataset - extends UpdateMatcher { + extends UpdateMatcher { public AbstractEnrichMissingDataset(final Topic topic) { super(true, rel -> topic, (p, rel) -> p.getDatasets().add(rel), - rel -> rel.getInstances().get(0).getUrl()); + rel -> rel.getOriginalId()); } protected abstract boolean filterByType(String relType); @Override - protected final List findDifferences( - final ResultWithRelations source, - final ResultWithRelations target) { + protected final List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { final Set existingDatasets = target .getDatasets() .stream() .filter(rel -> filterByType(rel.getRelType())) - .map(RelatedDataset::getRelDataset) - .map(Dataset::getId) + .map(Dataset::getOriginalId) .collect(Collectors.toSet()); return source .getDatasets() .stream() .filter(rel -> filterByType(rel.getRelType())) - .map(RelatedDataset::getRelDataset) - .filter(d -> !existingDatasets.contains(d.getId())) - .map(ConversionUtils::oafDatasetToBrokerDataset) + .filter(d -> !existingDatasets.contains(d.getOriginalId())) .collect(Collectors.toList()); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java index ab4325c1e..fa5fde725 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMissingProject.java @@ -1,19 +1,15 @@ package eu.dnetlib.dhp.broker.oa.matchers.relatedProjects; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.broker.objects.Project; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -public class EnrichMissingProject - extends UpdateMatcher { +public class EnrichMissingProject extends UpdateMatcher { public EnrichMissingProject() { super(true, @@ -23,16 +19,11 @@ public class EnrichMissingProject } @Override - protected List findDifferences(final ResultWithRelations source, final ResultWithRelations target) { - if (source.getProjects().isEmpty()) { - return Arrays.asList(); + protected List findDifferences(final OpenaireBrokerResult source, final OpenaireBrokerResult target) { + if (target.getProjects().isEmpty()) { + return source.getProjects(); } else { - return target - .getProjects() - .stream() - .map(RelatedProject::getRelProject) - .map(ConversionUtils::oafProjectToBrokerProject) - .collect(Collectors.toList()); + return new ArrayList<>(); } } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java index 3bf23a36b..ca63aeb49 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedProjects/EnrichMoreProject.java @@ -5,39 +5,38 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import eu.dnetlib.broker.objects.Project; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedProject; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Project; -public class EnrichMoreProject extends UpdateMatcher { +public class EnrichMoreProject extends UpdateMatcher { public EnrichMoreProject() { super(true, prj -> Topic.ENRICH_MORE_PROJECT, (p, prj) -> p.getProjects().add(prj), - prj -> prj.getFunder() + "::" + prj.getFundingProgram() + prj.getCode()); + prj -> projectAsString(prj)); + } + + private static String projectAsString(final Project prj) { + return prj.getFunder() + "::" + prj.getFundingProgram() + "::" + prj.getCode(); } @Override - protected List findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { - final Set existingProjects = source + final Set existingProjects = target .getProjects() .stream() - .map(RelatedProject::getRelProject) - .map(Project::getId) + .map(EnrichMoreProject::projectAsString) .collect(Collectors.toSet()); - return target + return source .getProjects() .stream() - .map(RelatedProject::getRelProject) - .filter(p -> !existingProjects.contains(p.getId())) - .map(ConversionUtils::oafProjectToBrokerProject) + .filter(p -> !existingProjects.contains(projectAsString(p))) .collect(Collectors.toList()); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java index bba3e9648..300863949 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedPublications/AbstractEnrichMissingPublication.java @@ -5,21 +5,18 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import eu.dnetlib.broker.objects.Publication; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedPublication; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Publication; -public abstract class AbstractEnrichMissingPublication - extends UpdateMatcher { +public abstract class AbstractEnrichMissingPublication extends UpdateMatcher { public AbstractEnrichMissingPublication(final Topic topic) { super(true, rel -> topic, (p, rel) -> p.getPublications().add(rel), - rel -> rel.getInstances().get(0).getUrl()); + rel -> rel.getOriginalId()); } @@ -27,24 +24,21 @@ public abstract class AbstractEnrichMissingPublication @Override protected final List findDifferences( - final ResultWithRelations source, - final ResultWithRelations target) { + final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { final Set existingPublications = target .getPublications() .stream() .filter(rel -> filterByType(rel.getRelType())) - .map(RelatedPublication::getRelPublication) - .map(Publication::getId) + .map(Publication::getOriginalId) .collect(Collectors.toSet()); return source .getPublications() .stream() .filter(rel -> filterByType(rel.getRelType())) - .map(RelatedPublication::getRelPublication) - .filter(d -> !existingPublications.contains(d.getId())) - .map(ConversionUtils::oafResultToBrokerPublication) + .filter(p -> !existingPublications.contains(p.getOriginalId())) .collect(Collectors.toList()); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java index 6090939dc..76ae061e6 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMissingSoftware.java @@ -1,15 +1,12 @@ package eu.dnetlib.dhp.broker.oa.matchers.relatedSoftware; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftware; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; public class EnrichMissingSoftware extends UpdateMatcher { @@ -23,18 +20,13 @@ public class EnrichMissingSoftware @Override protected List findDifferences( - final ResultWithRelations source, - final ResultWithRelations target) { + final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { - if (source.getSoftwares().isEmpty()) { - return Arrays.asList(); + if (target.getSoftwares().isEmpty()) { + return source.getSoftwares(); } else { - return target - .getSoftwares() - .stream() - .map(RelatedSoftware::getRelSoftware) - .map(ConversionUtils::oafSoftwareToBrokerSoftware) - .collect(Collectors.toList()); + return new ArrayList<>(); } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java index ba422f436..ebd421b8e 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/relatedSoftware/EnrichMoreSoftware.java @@ -5,15 +5,12 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import eu.dnetlib.broker.objects.Software; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.RelatedSoftware; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Software; -public class EnrichMoreSoftware - extends UpdateMatcher { +public class EnrichMoreSoftware extends UpdateMatcher { public EnrichMoreSoftware() { super(true, @@ -24,22 +21,19 @@ public class EnrichMoreSoftware @Override protected List findDifferences( - final ResultWithRelations source, - final ResultWithRelations target) { + final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { final Set existingSoftwares = source .getSoftwares() .stream() - .map(RelatedSoftware::getRelSoftware) - .map(Software::getId) + .map(Software::getName) .collect(Collectors.toSet()); return target .getSoftwares() .stream() - .map(RelatedSoftware::getRelSoftware) - .filter(p -> !existingSoftwares.contains(p.getId())) - .map(ConversionUtils::oafSoftwareToBrokerSoftware) + .filter(p -> !existingSoftwares.contains(p.getName())) .collect(Collectors.toList()); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java index 25d5f9d8a..b2cbbce2c 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAbstract.java @@ -5,9 +5,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; public class EnrichMissingAbstract extends UpdateMatcher { @@ -19,13 +19,12 @@ public class EnrichMissingAbstract extends UpdateMatcher { } @Override - protected List findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { - if (isMissing(target.getResult().getDescription()) && !isMissing(source.getResult().getDescription())) { - return Arrays - .asList(source.getResult().getDescription().get(0).getValue()); + protected List findDifferences(final OpenaireBrokerResult source, final OpenaireBrokerResult target) { + if (isMissing(target.getAbstracts()) && !isMissing(source.getAbstracts())) { + return Arrays.asList(source.getAbstracts().get(0)); + } else { + return new ArrayList<>(); } - return new ArrayList<>(); } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java index eed2d0be8..c4b96e67b 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingAuthorOrcid.java @@ -1,53 +1,43 @@ package eu.dnetlib.dhp.broker.oa.matchers.simple; -import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; + +import eu.dnetlib.broker.objects.Author; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Author; -import eu.dnetlib.dhp.schema.oaf.StructuredProperty; -public class EnrichMissingAuthorOrcid extends UpdateMatcher { +public class EnrichMissingAuthorOrcid extends UpdateMatcher { public EnrichMissingAuthorOrcid() { super(true, aut -> Topic.ENRICH_MISSING_AUTHOR_ORCID, (p, aut) -> p.getCreators().add(aut), - aut -> aut); + aut -> aut.getOrcid()); } @Override - protected List findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { final Set existingOrcids = target - .getResult() - .getAuthor() + .getCreators() .stream() - .map(Author::getPid) - .flatMap(List::stream) - .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) - .map(pid -> pid.getValue()) + .map(Author::getOrcid) + .filter(StringUtils::isNotBlank) .collect(Collectors.toSet()); - final List list = new ArrayList<>(); + return source + .getCreators() + .stream() + .filter(a -> StringUtils.isNotBlank(a.getOrcid())) + .filter(a -> !existingOrcids.contains(a.getOrcid())) + .collect(Collectors.toList()); - for (final Author author : source.getResult().getAuthor()) { - final String name = author.getFullname(); - - for (final StructuredProperty pid : author.getPid()) { - if (pid.getQualifier().getClassid().equalsIgnoreCase("orcid") - && !existingOrcids.contains(pid.getValue())) { - list.add(name + " [ORCID: " + pid.getValue() + "]"); - } - } - } - - return list; } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java index e8ca18ae9..e870cf1fa 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingOpenAccess.java @@ -6,11 +6,10 @@ import java.util.List; import java.util.stream.Collectors; import eu.dnetlib.broker.objects.Instance; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; public class EnrichMissingOpenAccess extends UpdateMatcher { @@ -22,13 +21,12 @@ public class EnrichMissingOpenAccess extends UpdateMatcher { } @Override - protected List findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { final long count = target - .getResult() - .getInstance() + .getInstances() .stream() - .map(i -> i.getAccessright().getClassid()) + .map(Instance::getLicense) .filter(right -> right.equals(BrokerConstants.OPEN_ACCESS)) .count(); @@ -37,12 +35,9 @@ public class EnrichMissingOpenAccess extends UpdateMatcher { } return source - .getResult() - .getInstance() + .getInstances() .stream() - .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(List::stream) + .filter(i -> i.getLicense().equals(BrokerConstants.OPEN_ACCESS)) .collect(Collectors.toList()); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java index 6c061a2e8..cc72d9fa9 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPid.java @@ -5,13 +5,12 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import eu.dnetlib.broker.objects.Pid; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import eu.dnetlib.broker.objects.TypedValue; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -public class EnrichMissingPid extends UpdateMatcher { +public class EnrichMissingPid extends UpdateMatcher { public EnrichMissingPid() { super(true, @@ -21,19 +20,17 @@ public class EnrichMissingPid extends UpdateMatcher { } @Override - protected List findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { - final long count = target.getResult().getPid().size(); + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { + final long count = target.getPids().size(); if (count > 0) { return Arrays.asList(); } return source - .getResult() - .getPid() + .getPids() .stream() - .map(ConversionUtils::oafPidToBrokerPid) .collect(Collectors.toList()); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java index 27a71740c..ed8c26b5a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingPublicationDate.java @@ -5,9 +5,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; public class EnrichMissingPublicationDate extends UpdateMatcher { @@ -19,13 +19,14 @@ public class EnrichMissingPublicationDate extends UpdateMatcher { } @Override - protected List findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { - if (isMissing(target.getResult().getDateofacceptance()) - && !isMissing(source.getResult().getDateofacceptance())) { - return Arrays.asList(source.getResult().getDateofacceptance().getValue()); + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { + + if (isMissing(target.getPublicationdate()) && !isMissing(source.getPublicationdate())) { + return Arrays.asList(source.getPublicationdate()); + } else { + return new ArrayList<>(); } - return new ArrayList<>(); } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java index 04de30089..07b1fa41a 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMissingSubject.java @@ -5,42 +5,38 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import eu.dnetlib.broker.objects.TypedValue; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Qualifier; -import eu.dnetlib.dhp.schema.oaf.StructuredProperty; -public class EnrichMissingSubject extends UpdateMatcher> { +public class EnrichMissingSubject extends UpdateMatcher { public EnrichMissingSubject() { super(true, - pair -> Topic.fromPath("ENRICH/MISSING/SUBJECT/" + pair.getLeft()), - (p, pair) -> p.getSubjects().add(pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight()); + s -> Topic.fromPath("ENRICH/MISSING/SUBJECT/" + s.getType()), + (p, s) -> p.getSubjects().add(s), + s -> subjectAsString(s)); } @Override - protected List> findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { - final Set existingTypes = target - .getResult() - .getSubject() + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { + final Set existingSubject = target + .getSubjects() .stream() - .map(StructuredProperty::getQualifier) - .map(Qualifier::getClassid) + .map(s -> subjectAsString(s)) .collect(Collectors.toSet()); return source - .getResult() - .getPid() + .getSubjects() .stream() - .filter(pid -> !existingTypes.contains(pid.getQualifier().getClassid())) - .map(ConversionUtils::oafSubjectToPair) + .filter(s -> !existingSubject.contains(subjectAsString(s))) .collect(Collectors.toList()); } + private static String subjectAsString(final TypedValue s) { + return s.getType() + "::" + s.getValue(); + } + } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java index 3ee4c8bdd..bfef3ee4f 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreOpenAccess.java @@ -6,11 +6,10 @@ import java.util.Set; import java.util.stream.Collectors; import eu.dnetlib.broker.objects.Instance; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.util.BrokerConstants; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; public class EnrichMoreOpenAccess extends UpdateMatcher { @@ -22,24 +21,19 @@ public class EnrichMoreOpenAccess extends UpdateMatcher { } @Override - protected List findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { final Set urls = target - .getResult() - .getInstance() + .getInstances() .stream() - .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) + .filter(i -> i.getLicense().equals(BrokerConstants.OPEN_ACCESS)) .map(i -> i.getUrl()) - .flatMap(List::stream) .collect(Collectors.toSet()); return source - .getResult() - .getInstance() + .getInstances() .stream() - .filter(i -> i.getAccessright().getClassid().equals(BrokerConstants.OPEN_ACCESS)) - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(List::stream) + .filter(i -> i.getLicense().equals(BrokerConstants.OPEN_ACCESS)) .filter(i -> !urls.contains(i.getUrl())) .collect(Collectors.toList()); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java index 694c9460a..d1f2e6022 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMorePid.java @@ -5,38 +5,37 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import eu.dnetlib.broker.objects.Pid; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import eu.dnetlib.broker.objects.TypedValue; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -public class EnrichMorePid extends UpdateMatcher { +public class EnrichMorePid extends UpdateMatcher { public EnrichMorePid() { super(true, pid -> Topic.ENRICH_MORE_PID, (p, pid) -> p.getPids().add(pid), - pid -> pid.getType() + "::" + pid.getValue()); + pid -> pidAsString(pid)); } @Override - protected List findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { final Set existingPids = target - .getResult() - .getPid() + .getPids() .stream() - .map(pid -> pid.getQualifier().getClassid() + "::" + pid.getValue()) + .map(pid -> pidAsString(pid)) .collect(Collectors.toSet()); return source - .getResult() - .getPid() + .getPids() .stream() - .filter(pid -> !existingPids.contains(pid.getQualifier().getClassid() + "::" + pid.getValue())) - .map(ConversionUtils::oafPidToBrokerPid) + .filter(pid -> !existingPids.contains(pidAsString(pid))) .collect(Collectors.toList()); } + private static String pidAsString(final TypedValue pid) { + return pid.getType() + "::" + pid.getValue(); + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java index 7d0b1a65e..39225e8ab 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/matchers/simple/EnrichMoreSubject.java @@ -5,39 +5,37 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; - +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import eu.dnetlib.broker.objects.TypedValue; import eu.dnetlib.dhp.broker.model.Topic; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; -import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -public class EnrichMoreSubject extends UpdateMatcher> { +public class EnrichMoreSubject extends UpdateMatcher { public EnrichMoreSubject() { super(true, - pair -> Topic.fromPath("ENRICH/MORE/SUBJECT/" + pair.getLeft()), - (p, pair) -> p.getSubjects().add(pair.getRight()), - pair -> pair.getLeft() + "::" + pair.getRight()); + s -> Topic.fromPath("ENRICH/MORE/SUBJECT/" + s.getType()), + (p, s) -> p.getSubjects().add(s), + s -> subjectAsString(s)); } @Override - protected List> findDifferences(final ResultWithRelations source, - final ResultWithRelations target) { + protected List findDifferences(final OpenaireBrokerResult source, + final OpenaireBrokerResult target) { final Set existingSubjects = target - .getResult() - .getSubject() + .getSubjects() .stream() - .map(pid -> pid.getQualifier().getClassid() + "::" + pid.getValue()) + .map(pid -> subjectAsString(pid)) .collect(Collectors.toSet()); return source - .getResult() - .getPid() + .getPids() .stream() - .filter(pid -> !existingSubjects.contains(pid.getQualifier().getClassid() + "::" + pid.getValue())) - .map(ConversionUtils::oafSubjectToPair) + .filter(s -> !existingSubjects.contains(subjectAsString(s))) .collect(Collectors.toList()); } + private static String subjectAsString(final TypedValue s) { + return s.getType() + "::" + s.getValue(); + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java index 0665c69dd..49c46c7f0 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/BrokerConstants.java @@ -2,13 +2,12 @@ package eu.dnetlib.dhp.broker.oa.util; import java.util.Arrays; -import java.util.List; +import java.util.HashSet; +import java.util.Set; -import eu.dnetlib.dhp.schema.oaf.Dataset; -import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct; -import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.Software; +import eu.dnetlib.dhp.broker.model.Event; +import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; +import eu.dnetlib.dhp.schema.common.ModelSupport; public class BrokerConstants { @@ -18,7 +17,11 @@ public class BrokerConstants { public static final float MIN_TRUST = 0.25f; public static final float MAX_TRUST = 1.00f; - public static final List> RESULT_CLASSES = Arrays - .asList(Publication.class, Dataset.class, Software.class, OtherResearchProduct.class); + public static Class[] getModelClasses() { + final Set> list = new HashSet<>(); + list.addAll(Arrays.asList(ModelSupport.getOafModelClasses())); + list.addAll(Arrays.asList(ResultGroup.class, Event.class)); + return list.toArray(new Class[] {}); + } } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java index 2b39115b1..b80848682 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java @@ -13,7 +13,8 @@ import org.dom4j.DocumentHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.dnetlib.broker.objects.Pid; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import eu.dnetlib.broker.objects.TypedValue; import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.Dataset; import eu.dnetlib.dhp.schema.oaf.ExternalReference; @@ -41,8 +42,8 @@ public class ConversionUtils { }).collect(Collectors.toList()); } - public static Pid oafPidToBrokerPid(final StructuredProperty sp) { - return sp != null ? new Pid() + public static TypedValue oafPidToBrokerPid(final StructuredProperty sp) { + return sp != null ? new TypedValue() .setValue(sp.getValue()) .setType(sp.getQualifier().getClassid()) : null; } @@ -54,7 +55,7 @@ public class ConversionUtils { public static final eu.dnetlib.broker.objects.Dataset oafDatasetToBrokerDataset(final Dataset d) { return d != null ? new eu.dnetlib.broker.objects.Dataset() .setOriginalId(d.getOriginalId().get(0)) - .setTitles(structPropList(d.getTitle())) + .setTitle(structPropValue(d.getTitle())) .setPids(d.getPid().stream().map(ConversionUtils::oafPidToBrokerPid).collect(Collectors.toList())) .setInstances( d @@ -63,26 +64,30 @@ public class ConversionUtils { .map(ConversionUtils::oafInstanceToBrokerInstances) .flatMap(List::stream) .collect(Collectors.toList())) - .setCollectedFrom(d.getCollectedfrom().stream().map(KeyValue::getValue).collect(Collectors.toList())) + .setCollectedFrom(d.getCollectedfrom().stream().map(KeyValue::getValue).findFirst().orElse(null)) : null; } - public static final eu.dnetlib.broker.objects.Publication oafResultToBrokerPublication(final Result result) { + public static final OpenaireBrokerResult oafResultToBrokerResult(final Result result) { - return result != null ? new eu.dnetlib.broker.objects.Publication() + return result != null ? new OpenaireBrokerResult() + .setOpenaireId(result.getId()) .setOriginalId(result.getOriginalId().get(0)) + .setTypology(result.getResulttype().getClassid()) .setTitles(structPropList(result.getTitle())) .setAbstracts(fieldList(result.getDescription())) .setLanguage(result.getLanguage().getClassid()) - .setSubjects(structPropList(result.getSubject())) - .setCreators(result.getAuthor().stream().map(Author::getFullname).collect(Collectors.toList())) - .setPublicationdate(result.getDateofcollection()) + .setSubjects(structPropTypedList(result.getSubject())) + .setCreators( + result.getAuthor().stream().map(ConversionUtils::oafAuthorToBrokerAuthor).collect(Collectors.toList())) + .setPublicationdate(result.getDateofacceptance().getValue()) .setPublisher(fieldValue(result.getPublisher())) .setEmbargoenddate(fieldValue(result.getEmbargoenddate())) .setContributor(fieldList(result.getContributor())) .setJournal( result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null) - .setCollectedFrom(result.getCollectedfrom().stream().map(KeyValue::getValue).collect(Collectors.toList())) + .setCollectedFromId(result.getCollectedfrom().stream().map(KeyValue::getKey).findFirst().orElse(null)) + .setCollectedFromName(result.getCollectedfrom().stream().map(KeyValue::getValue).findFirst().orElse(null)) .setPids(result.getPid().stream().map(ConversionUtils::oafPidToBrokerPid).collect(Collectors.toList())) .setInstances( result @@ -100,6 +105,30 @@ public class ConversionUtils { : null; } + private static List structPropTypedList(final List list) { + return list + .stream() + .map( + p -> new TypedValue() + .setValue(p.getValue()) + .setType(p.getQualifier().getClassid())) + .collect(Collectors.toList()); + } + + private static eu.dnetlib.broker.objects.Author oafAuthorToBrokerAuthor(final Author author) { + return author != null ? new eu.dnetlib.broker.objects.Author() + .setFullname(author.getFullname()) + .setOrcid( + author + .getPid() + .stream() + .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) + .map(pid -> pid.getValue()) + .findFirst() + .orElse(null)) + : null; + } + private static eu.dnetlib.broker.objects.Journal oafJournalToBrokerJournal(final Journal journal) { return journal != null ? new eu.dnetlib.broker.objects.Journal() .setName(journal.getName()) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java index b4de08db7..4c20ac5ca 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/EventFinder.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.broker.oa.util; import java.util.ArrayList; import java.util.List; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.broker.model.EventFactory; import eu.dnetlib.dhp.broker.oa.matchers.UpdateMatcher; import eu.dnetlib.dhp.broker.oa.matchers.relatedDatasets.EnrichMissingDatasetIsReferencedBy; @@ -30,7 +31,6 @@ import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreOpenAccess; import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMorePid; import eu.dnetlib.dhp.broker.oa.matchers.simple.EnrichMoreSubject; import eu.dnetlib.dhp.broker.oa.util.aggregators.simple.ResultGroup; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; import eu.dnetlib.pace.config.DedupConfig; public class EventFinder { @@ -68,7 +68,7 @@ public class EventFinder { public static EventGroup generateEvents(final ResultGroup results, final DedupConfig dedupConfig) { final List> list = new ArrayList<>(); - for (final ResultWithRelations target : results.getData()) { + for (final OpenaireBrokerResult target : results.getData()) { for (final UpdateMatcher matcher : matchers) { list.addAll(matcher.searchUpdatesForRecord(target, results.getData(), dedupConfig)); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java index 9da636413..82d017864 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/UpdateInfo.java @@ -1,7 +1,6 @@ package eu.dnetlib.dhp.broker.oa.util; -import java.util.List; import java.util.function.BiConsumer; import java.util.function.Function; @@ -10,14 +9,11 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.broker.objects.Instance; import eu.dnetlib.broker.objects.OpenAireEventPayload; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.broker.objects.Provenance; -import eu.dnetlib.broker.objects.Publication; import eu.dnetlib.dhp.broker.model.Topic; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; -import eu.dnetlib.dhp.schema.oaf.Instance; -import eu.dnetlib.dhp.schema.oaf.KeyValue; -import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.pace.config.DedupConfig; import eu.dnetlib.pace.model.MapDocument; import eu.dnetlib.pace.tree.support.TreeProcessor; @@ -29,11 +25,11 @@ public final class UpdateInfo { private final T highlightValue; - private final ResultWithRelations source; + private final OpenaireBrokerResult source; - private final ResultWithRelations target; + private final OpenaireBrokerResult target; - private final BiConsumer compileHighlight; + private final BiConsumer compileHighlight; private final Function highlightToString; @@ -41,9 +37,9 @@ public final class UpdateInfo { private static final Logger log = LoggerFactory.getLogger(UpdateInfo.class); - public UpdateInfo(final Topic topic, final T highlightValue, final ResultWithRelations source, - final ResultWithRelations target, - final BiConsumer compileHighlight, + public UpdateInfo(final Topic topic, final T highlightValue, final OpenaireBrokerResult source, + final OpenaireBrokerResult target, + final BiConsumer compileHighlight, final Function highlightToString, final DedupConfig dedupConfig) { this.topic = topic; @@ -52,22 +48,23 @@ public final class UpdateInfo { this.target = target; this.compileHighlight = compileHighlight; this.highlightToString = highlightToString; - this.trust = calculateTrust(dedupConfig, source.getResult(), target.getResult()); + this.trust = calculateTrust(dedupConfig, source, target); } public T getHighlightValue() { return highlightValue; } - public ResultWithRelations getSource() { + public OpenaireBrokerResult getSource() { return source; } - public ResultWithRelations getTarget() { + public OpenaireBrokerResult getTarget() { return target; } - private float calculateTrust(final DedupConfig dedupConfig, final Result r1, final Result r2) { + private float calculateTrust(final DedupConfig dedupConfig, final OpenaireBrokerResult r1, + final OpenaireBrokerResult r2) { try { final ObjectMapper objectMapper = new ObjectMapper(); final MapDocument doc1 = MapDocumentUtil @@ -103,26 +100,18 @@ public final class UpdateInfo { public OpenAireEventPayload asBrokerPayload() { - final Publication p = ConversionUtils.oafResultToBrokerPublication(getSource().getResult()); - compileHighlight.accept(p, getHighlightValue()); + compileHighlight.accept(target, getHighlightValue()); - final Publication hl = new Publication(); + final OpenaireBrokerResult hl = new OpenaireBrokerResult(); compileHighlight.accept(hl, getHighlightValue()); - final String provId = getSource().getResult().getOriginalId().stream().findFirst().orElse(null); - final String provRepo = getSource() - .getResult() - .getCollectedfrom() - .stream() - .map(KeyValue::getValue) - .findFirst() - .orElse(null); + final String provId = getSource().getOriginalId(); + final String provRepo = getSource().getCollectedFromName(); + final String provUrl = getSource() - .getResult() - .getInstance() + .getInstances() .stream() .map(Instance::getUrl) - .flatMap(List::stream) .findFirst() .orElse(null); ; @@ -130,7 +119,7 @@ public final class UpdateInfo { final Provenance provenance = new Provenance().setId(provId).setRepositoryName(provRepo).setUrl(provUrl); return new OpenAireEventPayload() - .setPublication(p) + .setPublication(target) .setHighlight(hl) .setTrust(trust) .setProvenance(provenance); diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java index 397f30660..dabe2bb4d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultAggregator.java @@ -5,11 +5,11 @@ import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.expressions.Aggregator; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; import eu.dnetlib.dhp.schema.oaf.Relation; import scala.Tuple2; -public class ResultAggregator extends Aggregator, ResultGroup, ResultGroup> { +public class ResultAggregator extends Aggregator, ResultGroup, ResultGroup> { /** * @@ -22,7 +22,7 @@ public class ResultAggregator extends Aggregator t) { + public ResultGroup reduce(final ResultGroup group, final Tuple2 t) { return group.addElement(t._1); } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java index 81f189d62..4308224a5 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/simple/ResultGroup.java @@ -5,7 +5,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; -import eu.dnetlib.dhp.broker.oa.util.aggregators.withRels.ResultWithRelations; +import eu.dnetlib.broker.objects.OpenaireBrokerResult; public class ResultGroup implements Serializable { @@ -14,13 +14,13 @@ public class ResultGroup implements Serializable { */ private static final long serialVersionUID = -3360828477088669296L; - private final List data = new ArrayList<>(); + private final List data = new ArrayList<>(); - public List getData() { + public List getData() { return data; } - public ResultGroup addElement(final ResultWithRelations elem) { + public ResultGroup addElement(final OpenaireBrokerResult elem) { data.add(elem); return this; } diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/OpenaireBrokerResultAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/OpenaireBrokerResultAggregator.java new file mode 100644 index 000000000..b44fbe367 --- /dev/null +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/OpenaireBrokerResultAggregator.java @@ -0,0 +1,69 @@ + +package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; + +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.expressions.Aggregator; + +import eu.dnetlib.broker.objects.OpenaireBrokerResult; +import scala.Tuple2; + +public class OpenaireBrokerResultAggregator + extends Aggregator, OpenaireBrokerResult, OpenaireBrokerResult> { + + /** + * + */ + private static final long serialVersionUID = -3687878788861013488L; + + @Override + public OpenaireBrokerResult zero() { + return new OpenaireBrokerResult(); + } + + @Override + public OpenaireBrokerResult finish(final OpenaireBrokerResult g) { + return g; + } + + @Override + public OpenaireBrokerResult reduce(final OpenaireBrokerResult g, final Tuple2 t) { + if (g.getOriginalId() == null) { + return t._1; + } else if (t._2 instanceof RelatedSoftware) { + g.getSoftwares().add(((RelatedSoftware) t._2).getRelSoftware()); + } else if (t._2 instanceof RelatedDataset) { + g.getDatasets().add(((RelatedDataset) t._2).getRelDataset()); + } else if (t._2 instanceof RelatedPublication) { + g.getPublications().add(((RelatedPublication) t._2).getRelPublication()); + } else if (t._2 instanceof RelatedProject) { + g.getProjects().add(((RelatedProject) t._2).getRelProject()); + } + return g; + + } + + @Override + public OpenaireBrokerResult merge(final OpenaireBrokerResult g1, final OpenaireBrokerResult g2) { + if (g1.getOriginalId() != null) { + g1.getSoftwares().addAll(g2.getSoftwares()); + g1.getDatasets().addAll(g2.getDatasets()); + g1.getPublications().addAll(g2.getPublications()); + g1.getProjects().addAll(g2.getProjects()); + return g1; + } else { + return g2; + } + } + + @Override + public Encoder bufferEncoder() { + return Encoders.kryo(OpenaireBrokerResult.class); + } + + @Override + public Encoder outputEncoder() { + return Encoders.kryo(OpenaireBrokerResult.class); + } + +} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java index 84cf693ad..fcf1b89b1 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedDataset.java @@ -1,10 +1,16 @@ package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; -import eu.dnetlib.dhp.schema.oaf.Dataset; +import java.io.Serializable; -public class RelatedDataset { +import eu.dnetlib.broker.objects.Dataset; +public class RelatedDataset implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 774487705184038324L; private final String source; private final String relType; private final Dataset relDataset; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java index 490724a44..08d57a1ea 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java @@ -1,15 +1,17 @@ package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; -import eu.dnetlib.dhp.schema.oaf.Dataset; -import eu.dnetlib.dhp.schema.oaf.Project; -import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.dhp.schema.oaf.Software; +import eu.dnetlib.broker.objects.Dataset; +import eu.dnetlib.broker.objects.Project; +import eu.dnetlib.broker.objects.Publication; +import eu.dnetlib.broker.objects.Software; public class RelatedEntityFactory { @SuppressWarnings("unchecked") - public static RT newRelatedEntity(final String sourceId, final String relType, final T target, + public static RT newRelatedEntity(final String sourceId, + final String relType, + final T target, final Class clazz) { if (clazz == RelatedProject.class) { return (RT) new RelatedProject(sourceId, relType, (Project) target); diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java index 2d16f5409..233041c09 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedProject.java @@ -1,9 +1,16 @@ package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; -import eu.dnetlib.dhp.schema.oaf.Project; +import java.io.Serializable; -public class RelatedProject { +import eu.dnetlib.broker.objects.Project; + +public class RelatedProject implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 4941437626549329870L; private final String source; private final String relType; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java index f1545c004..80b92462d 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedPublication.java @@ -1,9 +1,16 @@ package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; -import eu.dnetlib.dhp.schema.oaf.Publication; +import java.io.Serializable; -public class RelatedPublication { +import eu.dnetlib.broker.objects.Publication; + +public class RelatedPublication implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 9021609640411395128L; private final String source; private final String relType; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java index e5873d263..13f1f4290 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedSoftware.java @@ -1,10 +1,16 @@ package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; -import eu.dnetlib.dhp.schema.oaf.Software; +import java.io.Serializable; -public class RelatedSoftware { +import eu.dnetlib.broker.objects.Software; +public class RelatedSoftware implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 7573383356943300157L; private final String source; private final String relType; private final Software relSoftware; diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelations.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelations.java deleted file mode 100644 index 2d762aded..000000000 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelations.java +++ /dev/null @@ -1,55 +0,0 @@ - -package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import eu.dnetlib.dhp.schema.oaf.Result; - -public class ResultWithRelations implements Serializable { - - /** - * - */ - private static final long serialVersionUID = -1368401915974311571L; - - private Result result; - - private final List datasets = new ArrayList<>(); - private final List publications = new ArrayList<>(); - private final List softwares = new ArrayList<>(); - private final List projects = new ArrayList<>(); - - public ResultWithRelations() { - } - - public ResultWithRelations(final Result result) { - this.result = result; - } - - public Result getResult() { - return result; - } - - public List getDatasets() { - return datasets; - } - - public List getPublications() { - return publications; - } - - public List getSoftwares() { - return softwares; - } - - public List getProjects() { - return projects; - } - - public void setResult(final Result result) { - this.result = result; - } - -} diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelationsAggregator.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelationsAggregator.java deleted file mode 100644 index b4922a64f..000000000 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/ResultWithRelationsAggregator.java +++ /dev/null @@ -1,68 +0,0 @@ - -package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; - -import org.apache.spark.sql.Encoder; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.expressions.Aggregator; - -import scala.Tuple2; - -public class ResultWithRelationsAggregator - extends Aggregator, ResultWithRelations, ResultWithRelations> { - - /** - * - */ - private static final long serialVersionUID = -3687878788861013488L; - - @Override - public ResultWithRelations zero() { - return new ResultWithRelations(); - } - - @Override - public ResultWithRelations finish(final ResultWithRelations g) { - return g; - } - - @Override - public ResultWithRelations reduce(final ResultWithRelations g, final Tuple2 t) { - if (g.getResult() == null) { - return t._1; - } else if (t._2 instanceof RelatedSoftware) { - g.getSoftwares().add((RelatedSoftware) t._2); - } else if (t._2 instanceof RelatedDataset) { - g.getDatasets().add((RelatedDataset) t._2); - } else if (t._2 instanceof RelatedPublication) { - g.getPublications().add((RelatedPublication) t._2); - } else if (t._2 instanceof RelatedProject) { - g.getProjects().add((RelatedProject) t._2); - } - return g; - - } - - @Override - public ResultWithRelations merge(final ResultWithRelations g1, final ResultWithRelations g2) { - if (g1.getResult() != null) { - g1.getSoftwares().addAll(g2.getSoftwares()); - g1.getDatasets().addAll(g2.getDatasets()); - g1.getPublications().addAll(g2.getPublications()); - g1.getProjects().addAll(g2.getProjects()); - return g1; - } else { - return g2; - } - } - - @Override - public Encoder bufferEncoder() { - return Encoders.kryo(ResultWithRelations.class); - } - - @Override - public Encoder outputEncoder() { - return Encoders.kryo(ResultWithRelations.class); - } - -} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/reflections/ReflectionTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/reflections/ReflectionTest.java new file mode 100644 index 000000000..110fabf45 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/reflections/ReflectionTest.java @@ -0,0 +1,200 @@ + +package eu.dnetlib.dhp.oa.graph.reflections; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +public class ReflectionTest { + + private final Cleaner cleaner = new Cleaner(); + + @Test + void testObject() throws Exception { + final Publication pub = new Publication(); + pub.setTitle("openaire guidelines"); + pub.getAuthors().add(new Author("Michele Artini", new Prop("aa-001", "orcid"))); + pub.getAuthors().add(new Author("Claudio Atzori", new Prop("aa-002", "orcid"))); + pub.getAuthors().add(new Author("Alessia Bardi", new Prop("aa-003", "orcid"))); + pub.getSubjects().add(new Prop("infrastructures", "keyword")); + pub.getSubjects().add(new Prop("digital libraries", "keyword")); + + cleaner.clean(pub); + + System.out.println(pub); + + assertEquals("OPENAIRE GUIDELINES", pub.getTitle()); + + assertEquals("MICHELE ARTINI", pub.getAuthors().get(0).getName()); + assertEquals("CLAUDIO ATZORI", pub.getAuthors().get(1).getName()); + assertEquals("ALESSIA BARDI", pub.getAuthors().get(2).getName()); + + assertEquals("dnet:aa-001", pub.getAuthors().get(0).getId().getId()); + assertEquals("dnet:aa-002", pub.getAuthors().get(1).getId().getId()); + assertEquals("dnet:aa-003", pub.getAuthors().get(2).getId().getId()); + assertEquals("dnet:orcid", pub.getAuthors().get(0).getId().getName()); + assertEquals("dnet:orcid", pub.getAuthors().get(1).getId().getName()); + assertEquals("dnet:orcid", pub.getAuthors().get(2).getId().getName()); + + assertEquals("dnet:infrastructures", pub.getSubjects().get(0).getId()); + assertEquals("dnet:keyword", pub.getSubjects().get(0).getName()); + assertEquals("dnet:digital libraries", pub.getSubjects().get(1).getId()); + assertEquals("dnet:keyword", pub.getSubjects().get(1).getName()); + } + +} + +class Cleaner { + + public void clean(final Object o) throws IllegalArgumentException, IllegalAccessException { + if (isPrimitive(o)) { + return; + } else if (isIterable(o.getClass())) { + for (final Object elem : (Iterable) o) { + clean(elem); + } + } else if (hasMapping(o)) { + mapObject(o); + } else { + for (final Field f : o.getClass().getDeclaredFields()) { + f.setAccessible(true); + final Object val = f.get(o); + if (isPrimitive(val)) { + f.set(o, cleanValue(f.get(o))); + } else if (hasMapping(val)) { + mapObject(val); + } else { + clean(f.get(o)); + } + } + } + } + + private boolean hasMapping(final Object o) { + return o.getClass() == Prop.class; + } + + private void mapObject(final Object o) { + if (o.getClass() == Prop.class) { + ((Prop) o).setId("dnet:" + ((Prop) o).getId()); + ((Prop) o).setName("dnet:" + ((Prop) o).getName()); + } + + } + + private Object cleanValue(final Object o) { + if (o.getClass() == String.class) { + return ((String) o).toUpperCase(); + } else { + return o; + } + + } + + private boolean isIterable(final Class cl) { + return Iterable.class.isAssignableFrom(cl); + } + + private boolean isPrimitive(final Object o) { + return o.getClass() == String.class; + } +} + +class Publication { + + private String title; + private final List authors = new ArrayList<>(); + private final List subjects = new ArrayList<>(); + + public String getTitle() { + return title; + } + + public void setTitle(final String title) { + this.title = title; + } + + public List getAuthors() { + return authors; + } + + public List getSubjects() { + return subjects; + } + + @Override + public String toString() { + return String.format("Publication [title=%s, authors=%s, subjects=%s]", title, authors, subjects); + } + +} + +class Prop { + + private String id; + private String name; + + public Prop(final String id, final String name) { + this.id = id; + this.name = name; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + @Override + public String toString() { + return String.format("Prop [id=%s, name=%s]", id, name); + } + +} + +class Author { + + private String name; + private Prop id; + + public Author(final String name, final Prop id) { + this.name = name; + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public Prop getId() { + return id; + } + + public void setId(final Prop id) { + this.id = id; + } + + @Override + public String toString() { + return String.format("Author [name=%s, id=%s]", name, id); + } + +} From 4ec262db5318661bcfba6cb9497cfcca3370bbbc Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 16 Jun 2020 15:28:20 +0200 Subject: [PATCH 82/88] included externalreference(s) in the result view on the Hive graph DB --- .../graph/hive/oozie_app/lib/scripts/postprocessing.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/lib/scripts/postprocessing.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/lib/scripts/postprocessing.sql index 6c49679cd..7bec2fe04 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/lib/scripts/postprocessing.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/lib/scripts/postprocessing.sql @@ -1,10 +1,10 @@ DROP VIEW IF EXISTS ${hiveDbName}.result; CREATE VIEW IF NOT EXISTS result as - select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hiveDbName}.publication p + select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, externalreference, instance from ${hiveDbName}.publication p union all - select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hiveDbName}.dataset d + select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, externalreference, instance from ${hiveDbName}.dataset d union all - select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hiveDbName}.software s + select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, externalreference, instance from ${hiveDbName}.software s union all - select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, instance from ${hiveDbName}.otherresearchproduct o; + select id, dateofcollection, title, publisher, bestaccessright, datainfo, collectedfrom, pid, author, resulttype, language, country, subject, description, dateofacceptance, embargoenddate, resourcetype, context, externalreference, instance from ${hiveDbName}.otherresearchproduct o; From 76ea7607f723788f5d390bc5cd08546fd152091f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 16 Jun 2020 15:53:13 +0200 Subject: [PATCH 83/88] partial refactoring --- .../broker/oa/GenerateEventsApplication.java | 46 ++++------ .../dhp/broker/oa/util/ConversionUtils.java | 84 ++++++++++--------- .../withRels/RelatedEntityFactory.java | 30 +++---- 3 files changed, 76 insertions(+), 84 deletions(-) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java index 940d7f9f3..5efaed784 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java @@ -51,9 +51,8 @@ public class GenerateEventsApplication { public static void main(final String[] args) throws Exception { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils - .toString( - GenerateEventsApplication.class - .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/generate_broker_events.json"))); + .toString(GenerateEventsApplication.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/generate_broker_events.json"))); parser.parseArgument(args); final Boolean isSparkSessionManaged = Optional @@ -118,14 +117,11 @@ public class GenerateEventsApplication { return results .joinWith(mergedRels, results.col("result.id").equalTo(mergedRels.col("source")), "inner") - .groupByKey( - (MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) + .groupByKey((MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) .agg(aggr) .map((MapFunction, ResultGroup>) t -> t._2, Encoders.kryo(ResultGroup.class)) .filter(ResultGroup::isValid) - .map( - (MapFunction) g -> EventFinder.generateEvents(g, dedupConfig), - Encoders.kryo(EventGroup.class)) + .map((MapFunction) g -> EventFinder.generateEvents(g, dedupConfig), Encoders.kryo(EventGroup.class)) .flatMap(group -> group.getData().iterator(), Encoders.kryo(Event.class)); } @@ -134,25 +130,22 @@ public class GenerateEventsApplication { final String graphPath, final Class sourceClass) { final Dataset projects = readPath(spark, graphPath + "/project", Project.class); - final Dataset datasets = readPath( - spark, graphPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class); + final Dataset datasets = readPath(spark, graphPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class); final Dataset softwares = readPath(spark, graphPath + "/software", Software.class); final Dataset publications = readPath(spark, graphPath + "/publication", Publication.class); final Dataset rels = readPath(spark, graphPath + "/relation", Relation.class) - .filter(r -> !r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)); + .filter(r -> !r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)) + .cache(); - final Dataset r0 = readPath( - spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) - .filter(r -> r.getDataInfo().getDeletedbyinference()) - .map(ConversionUtils::oafResultToBrokerResult, Encoders.kryo(OpenaireBrokerResult.class)); + final Dataset r0 = readPath(spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()) + .map(ConversionUtils::oafResultToBrokerResult, Encoders.kryo(OpenaireBrokerResult.class)); final Dataset r1 = join(r0, rels, relatedEntities(projects, rels, RelatedProject.class)); final Dataset r2 = join(r1, rels, relatedEntities(softwares, rels, RelatedProject.class)); final Dataset r3 = join(r2, rels, relatedEntities(datasets, rels, RelatedProject.class)); - final Dataset r4 = join( - r3, rels, relatedEntities(publications, rels, RelatedProject.class)); - ; + final Dataset r4 = join(r3, rels, relatedEntities(publications, rels, RelatedProject.class));; return r4; } @@ -162,9 +155,7 @@ public class GenerateEventsApplication { final Class clazz) { return rels .joinWith(targets, targets.col("id").equalTo(rels.col("target")), "inner") - .map( - t -> RelatedEntityFactory.newRelatedEntity(t._1.getSource(), t._1.getRelType(), t._2, clazz), - Encoders.kryo(clazz)); + .map(t -> RelatedEntityFactory.newRelatedEntity(t._1.getSource(), t._1.getRelType(), t._2, clazz), Encoders.kryo(clazz)); } private static Dataset join(final Dataset sources, @@ -172,13 +163,11 @@ public class GenerateEventsApplication { final Dataset typedRels) { final TypedColumn, OpenaireBrokerResult> aggr = new OpenaireBrokerResultAggregator() - .toColumn(); - ; + .toColumn();; return sources .joinWith(typedRels, sources.col("result.id").equalTo(rels.col("source")), "left_outer") - .groupByKey( - (MapFunction, String>) t -> t._1.getOpenaireId(), Encoders.STRING()) + .groupByKey((MapFunction, String>) t -> t._1.getOpenaireId(), Encoders.STRING()) .agg(aggr) .map(t -> t._2, Encoders.kryo(OpenaireBrokerResult.class)); } @@ -197,11 +186,8 @@ public class GenerateEventsApplication { final ISLookUpService isLookUpService = ISLookupClientFactory.getLookUpService(isLookupUrl); final String conf = isLookUpService - .getResourceProfileByQuery( - String - .format( - "for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()", - profId)); + .getResourceProfileByQuery(String + .format("for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()", profId)); final DedupConfig dedupConfig = new ObjectMapper().readValue(conf, DedupConfig.class); dedupConfig.getPace().initModel(); diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java index b80848682..3693f4ce4 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java @@ -57,17 +57,31 @@ public class ConversionUtils { .setOriginalId(d.getOriginalId().get(0)) .setTitle(structPropValue(d.getTitle())) .setPids(d.getPid().stream().map(ConversionUtils::oafPidToBrokerPid).collect(Collectors.toList())) - .setInstances( - d - .getInstance() - .stream() - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(List::stream) - .collect(Collectors.toList())) + .setInstances(d + .getInstance() + .stream() + .map(ConversionUtils::oafInstanceToBrokerInstances) + .flatMap(List::stream) + .collect(Collectors.toList())) .setCollectedFrom(d.getCollectedfrom().stream().map(KeyValue::getValue).findFirst().orElse(null)) : null; } + public static eu.dnetlib.broker.objects.Publication oafPublicationToBrokerPublication(final Publication p) { + return p != null ? new eu.dnetlib.broker.objects.Publication() + .setOriginalId(p.getOriginalId().get(0)) + .setTitle(structPropValue(p.getTitle())) + .setPids(p.getPid().stream().map(ConversionUtils::oafPidToBrokerPid).collect(Collectors.toList())) + .setInstances(p + .getInstance() + .stream() + .map(ConversionUtils::oafInstanceToBrokerInstances) + .flatMap(List::stream) + .collect(Collectors.toList())) + .setCollectedFrom(p.getCollectedfrom().stream().map(KeyValue::getValue).findFirst().orElse(null)) + : null; + } + public static final OpenaireBrokerResult oafResultToBrokerResult(final Result result) { return result != null ? new OpenaireBrokerResult() @@ -78,54 +92,48 @@ public class ConversionUtils { .setAbstracts(fieldList(result.getDescription())) .setLanguage(result.getLanguage().getClassid()) .setSubjects(structPropTypedList(result.getSubject())) - .setCreators( - result.getAuthor().stream().map(ConversionUtils::oafAuthorToBrokerAuthor).collect(Collectors.toList())) + .setCreators(result.getAuthor().stream().map(ConversionUtils::oafAuthorToBrokerAuthor).collect(Collectors.toList())) .setPublicationdate(result.getDateofacceptance().getValue()) .setPublisher(fieldValue(result.getPublisher())) .setEmbargoenddate(fieldValue(result.getEmbargoenddate())) .setContributor(fieldList(result.getContributor())) - .setJournal( - result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null) + .setJournal(result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null) .setCollectedFromId(result.getCollectedfrom().stream().map(KeyValue::getKey).findFirst().orElse(null)) .setCollectedFromName(result.getCollectedfrom().stream().map(KeyValue::getValue).findFirst().orElse(null)) .setPids(result.getPid().stream().map(ConversionUtils::oafPidToBrokerPid).collect(Collectors.toList())) - .setInstances( - result - .getInstance() - .stream() - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(List::stream) - .collect(Collectors.toList())) - .setExternalReferences( - result - .getExternalReference() - .stream() - .map(ConversionUtils::oafExtRefToBrokerExtRef) - .collect(Collectors.toList())) + .setInstances(result + .getInstance() + .stream() + .map(ConversionUtils::oafInstanceToBrokerInstances) + .flatMap(List::stream) + .collect(Collectors.toList())) + .setExternalReferences(result + .getExternalReference() + .stream() + .map(ConversionUtils::oafExtRefToBrokerExtRef) + .collect(Collectors.toList())) : null; } private static List structPropTypedList(final List list) { return list .stream() - .map( - p -> new TypedValue() - .setValue(p.getValue()) - .setType(p.getQualifier().getClassid())) + .map(p -> new TypedValue() + .setValue(p.getValue()) + .setType(p.getQualifier().getClassid())) .collect(Collectors.toList()); } private static eu.dnetlib.broker.objects.Author oafAuthorToBrokerAuthor(final Author author) { return author != null ? new eu.dnetlib.broker.objects.Author() .setFullname(author.getFullname()) - .setOrcid( - author - .getPid() - .stream() - .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) - .map(pid -> pid.getValue()) - .findFirst() - .orElse(null)) + .setOrcid(author + .getPid() + .stream() + .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) + .map(pid -> pid.getValue()) + .findFirst() + .orElse(null)) : null; } @@ -147,9 +155,7 @@ public class ConversionUtils { } public static final eu.dnetlib.broker.objects.Project oafProjectToBrokerProject(final Project p) { - if (p == null) { - return null; - } + if (p == null) { return null; } final eu.dnetlib.broker.objects.Project res = new eu.dnetlib.broker.objects.Project() .setTitle(fieldValue(p.getTitle())) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java index 08d57a1ea..88dab2ba3 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java @@ -1,10 +1,11 @@ package eu.dnetlib.dhp.broker.oa.util.aggregators.withRels; -import eu.dnetlib.broker.objects.Dataset; -import eu.dnetlib.broker.objects.Project; -import eu.dnetlib.broker.objects.Publication; -import eu.dnetlib.broker.objects.Software; +import eu.dnetlib.dhp.broker.oa.util.ConversionUtils; +import eu.dnetlib.dhp.schema.oaf.Dataset; +import eu.dnetlib.dhp.schema.oaf.Project; +import eu.dnetlib.dhp.schema.oaf.Publication; +import eu.dnetlib.dhp.schema.oaf.Software; public class RelatedEntityFactory { @@ -13,18 +14,17 @@ public class RelatedEntityFactory { final String relType, final T target, final Class clazz) { + if (clazz == RelatedProject.class) { - return (RT) new RelatedProject(sourceId, relType, (Project) target); + return (RT) new RelatedProject(sourceId, relType, ConversionUtils.oafProjectToBrokerProject((Project) target)); + } else if (clazz == RelatedSoftware.class) { + return (RT) new RelatedSoftware(sourceId, relType, ConversionUtils.oafSoftwareToBrokerSoftware((Software) target)); + } else if (clazz == RelatedDataset.class) { + return (RT) new RelatedDataset(sourceId, relType, ConversionUtils.oafDatasetToBrokerDataset((Dataset) target)); + } else if (clazz == RelatedPublication.class) { + return (RT) new RelatedPublication(sourceId, relType, ConversionUtils.oafPublicationToBrokerPublication((Publication) target)); + } else { + return null; } - if (clazz == RelatedSoftware.class) { - return (RT) new RelatedSoftware(sourceId, relType, (Software) target); - } - if (clazz == RelatedDataset.class) { - return (RT) new RelatedDataset(sourceId, relType, (Dataset) target); - } - if (clazz == RelatedPublication.class) { - return (RT) new RelatedPublication(sourceId, relType, (Publication) target); - } - return null; } } From 9e2c23e39190647b9f3f832e28d3bb4f4f9e3a11 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 16 Jun 2020 15:55:42 +0200 Subject: [PATCH 84/88] partial refactoring --- .../broker/oa/GenerateEventsApplication.java | 43 ++++++---- .../dhp/broker/oa/util/ConversionUtils.java | 82 +++++++++++-------- .../withRels/RelatedEntityFactory.java | 12 ++- 3 files changed, 83 insertions(+), 54 deletions(-) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java index 5efaed784..c1f12f43c 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateEventsApplication.java @@ -51,8 +51,9 @@ public class GenerateEventsApplication { public static void main(final String[] args) throws Exception { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils - .toString(GenerateEventsApplication.class - .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/generate_broker_events.json"))); + .toString( + GenerateEventsApplication.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/generate_broker_events.json"))); parser.parseArgument(args); final Boolean isSparkSessionManaged = Optional @@ -117,11 +118,14 @@ public class GenerateEventsApplication { return results .joinWith(mergedRels, results.col("result.id").equalTo(mergedRels.col("source")), "inner") - .groupByKey((MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) + .groupByKey( + (MapFunction, String>) t -> t._2.getTarget(), Encoders.STRING()) .agg(aggr) .map((MapFunction, ResultGroup>) t -> t._2, Encoders.kryo(ResultGroup.class)) .filter(ResultGroup::isValid) - .map((MapFunction) g -> EventFinder.generateEvents(g, dedupConfig), Encoders.kryo(EventGroup.class)) + .map( + (MapFunction) g -> EventFinder.generateEvents(g, dedupConfig), + Encoders.kryo(EventGroup.class)) .flatMap(group -> group.getData().iterator(), Encoders.kryo(Event.class)); } @@ -130,7 +134,8 @@ public class GenerateEventsApplication { final String graphPath, final Class sourceClass) { final Dataset projects = readPath(spark, graphPath + "/project", Project.class); - final Dataset datasets = readPath(spark, graphPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class); + final Dataset datasets = readPath( + spark, graphPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class); final Dataset softwares = readPath(spark, graphPath + "/software", Software.class); final Dataset publications = readPath(spark, graphPath + "/publication", Publication.class); @@ -138,14 +143,17 @@ public class GenerateEventsApplication { .filter(r -> !r.getRelClass().equals(BrokerConstants.IS_MERGED_IN_CLASS)) .cache(); - final Dataset r0 = readPath(spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) - .filter(r -> r.getDataInfo().getDeletedbyinference()) - .map(ConversionUtils::oafResultToBrokerResult, Encoders.kryo(OpenaireBrokerResult.class)); + final Dataset r0 = readPath( + spark, graphPath + "/" + sourceClass.getSimpleName().toLowerCase(), Result.class) + .filter(r -> r.getDataInfo().getDeletedbyinference()) + .map(ConversionUtils::oafResultToBrokerResult, Encoders.kryo(OpenaireBrokerResult.class)); final Dataset r1 = join(r0, rels, relatedEntities(projects, rels, RelatedProject.class)); final Dataset r2 = join(r1, rels, relatedEntities(softwares, rels, RelatedProject.class)); final Dataset r3 = join(r2, rels, relatedEntities(datasets, rels, RelatedProject.class)); - final Dataset r4 = join(r3, rels, relatedEntities(publications, rels, RelatedProject.class));; + final Dataset r4 = join( + r3, rels, relatedEntities(publications, rels, RelatedProject.class)); + ; return r4; } @@ -155,7 +163,9 @@ public class GenerateEventsApplication { final Class clazz) { return rels .joinWith(targets, targets.col("id").equalTo(rels.col("target")), "inner") - .map(t -> RelatedEntityFactory.newRelatedEntity(t._1.getSource(), t._1.getRelType(), t._2, clazz), Encoders.kryo(clazz)); + .map( + t -> RelatedEntityFactory.newRelatedEntity(t._1.getSource(), t._1.getRelType(), t._2, clazz), + Encoders.kryo(clazz)); } private static Dataset join(final Dataset sources, @@ -163,11 +173,13 @@ public class GenerateEventsApplication { final Dataset typedRels) { final TypedColumn, OpenaireBrokerResult> aggr = new OpenaireBrokerResultAggregator() - .toColumn();; + .toColumn(); + ; return sources .joinWith(typedRels, sources.col("result.id").equalTo(rels.col("source")), "left_outer") - .groupByKey((MapFunction, String>) t -> t._1.getOpenaireId(), Encoders.STRING()) + .groupByKey( + (MapFunction, String>) t -> t._1.getOpenaireId(), Encoders.STRING()) .agg(aggr) .map(t -> t._2, Encoders.kryo(OpenaireBrokerResult.class)); } @@ -186,8 +198,11 @@ public class GenerateEventsApplication { final ISLookUpService isLookUpService = ISLookupClientFactory.getLookUpService(isLookupUrl); final String conf = isLookUpService - .getResourceProfileByQuery(String - .format("for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()", profId)); + .getResourceProfileByQuery( + String + .format( + "for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()", + profId)); final DedupConfig dedupConfig = new ObjectMapper().readValue(conf, DedupConfig.class); dedupConfig.getPace().initModel(); diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java index 3693f4ce4..d04ef45a0 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java @@ -57,12 +57,13 @@ public class ConversionUtils { .setOriginalId(d.getOriginalId().get(0)) .setTitle(structPropValue(d.getTitle())) .setPids(d.getPid().stream().map(ConversionUtils::oafPidToBrokerPid).collect(Collectors.toList())) - .setInstances(d - .getInstance() - .stream() - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(List::stream) - .collect(Collectors.toList())) + .setInstances( + d + .getInstance() + .stream() + .map(ConversionUtils::oafInstanceToBrokerInstances) + .flatMap(List::stream) + .collect(Collectors.toList())) .setCollectedFrom(d.getCollectedfrom().stream().map(KeyValue::getValue).findFirst().orElse(null)) : null; } @@ -72,12 +73,13 @@ public class ConversionUtils { .setOriginalId(p.getOriginalId().get(0)) .setTitle(structPropValue(p.getTitle())) .setPids(p.getPid().stream().map(ConversionUtils::oafPidToBrokerPid).collect(Collectors.toList())) - .setInstances(p - .getInstance() - .stream() - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(List::stream) - .collect(Collectors.toList())) + .setInstances( + p + .getInstance() + .stream() + .map(ConversionUtils::oafInstanceToBrokerInstances) + .flatMap(List::stream) + .collect(Collectors.toList())) .setCollectedFrom(p.getCollectedfrom().stream().map(KeyValue::getValue).findFirst().orElse(null)) : null; } @@ -92,48 +94,54 @@ public class ConversionUtils { .setAbstracts(fieldList(result.getDescription())) .setLanguage(result.getLanguage().getClassid()) .setSubjects(structPropTypedList(result.getSubject())) - .setCreators(result.getAuthor().stream().map(ConversionUtils::oafAuthorToBrokerAuthor).collect(Collectors.toList())) + .setCreators( + result.getAuthor().stream().map(ConversionUtils::oafAuthorToBrokerAuthor).collect(Collectors.toList())) .setPublicationdate(result.getDateofacceptance().getValue()) .setPublisher(fieldValue(result.getPublisher())) .setEmbargoenddate(fieldValue(result.getEmbargoenddate())) .setContributor(fieldList(result.getContributor())) - .setJournal(result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null) + .setJournal( + result instanceof Publication ? oafJournalToBrokerJournal(((Publication) result).getJournal()) : null) .setCollectedFromId(result.getCollectedfrom().stream().map(KeyValue::getKey).findFirst().orElse(null)) .setCollectedFromName(result.getCollectedfrom().stream().map(KeyValue::getValue).findFirst().orElse(null)) .setPids(result.getPid().stream().map(ConversionUtils::oafPidToBrokerPid).collect(Collectors.toList())) - .setInstances(result - .getInstance() - .stream() - .map(ConversionUtils::oafInstanceToBrokerInstances) - .flatMap(List::stream) - .collect(Collectors.toList())) - .setExternalReferences(result - .getExternalReference() - .stream() - .map(ConversionUtils::oafExtRefToBrokerExtRef) - .collect(Collectors.toList())) + .setInstances( + result + .getInstance() + .stream() + .map(ConversionUtils::oafInstanceToBrokerInstances) + .flatMap(List::stream) + .collect(Collectors.toList())) + .setExternalReferences( + result + .getExternalReference() + .stream() + .map(ConversionUtils::oafExtRefToBrokerExtRef) + .collect(Collectors.toList())) : null; } private static List structPropTypedList(final List list) { return list .stream() - .map(p -> new TypedValue() - .setValue(p.getValue()) - .setType(p.getQualifier().getClassid())) + .map( + p -> new TypedValue() + .setValue(p.getValue()) + .setType(p.getQualifier().getClassid())) .collect(Collectors.toList()); } private static eu.dnetlib.broker.objects.Author oafAuthorToBrokerAuthor(final Author author) { return author != null ? new eu.dnetlib.broker.objects.Author() .setFullname(author.getFullname()) - .setOrcid(author - .getPid() - .stream() - .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) - .map(pid -> pid.getValue()) - .findFirst() - .orElse(null)) + .setOrcid( + author + .getPid() + .stream() + .filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid")) + .map(pid -> pid.getValue()) + .findFirst() + .orElse(null)) : null; } @@ -155,7 +163,9 @@ public class ConversionUtils { } public static final eu.dnetlib.broker.objects.Project oafProjectToBrokerProject(final Project p) { - if (p == null) { return null; } + if (p == null) { + return null; + } final eu.dnetlib.broker.objects.Project res = new eu.dnetlib.broker.objects.Project() .setTitle(fieldValue(p.getTitle())) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java index 88dab2ba3..c60d4f141 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/aggregators/withRels/RelatedEntityFactory.java @@ -16,13 +16,17 @@ public class RelatedEntityFactory { final Class clazz) { if (clazz == RelatedProject.class) { - return (RT) new RelatedProject(sourceId, relType, ConversionUtils.oafProjectToBrokerProject((Project) target)); + return (RT) new RelatedProject(sourceId, relType, + ConversionUtils.oafProjectToBrokerProject((Project) target)); } else if (clazz == RelatedSoftware.class) { - return (RT) new RelatedSoftware(sourceId, relType, ConversionUtils.oafSoftwareToBrokerSoftware((Software) target)); + return (RT) new RelatedSoftware(sourceId, relType, + ConversionUtils.oafSoftwareToBrokerSoftware((Software) target)); } else if (clazz == RelatedDataset.class) { - return (RT) new RelatedDataset(sourceId, relType, ConversionUtils.oafDatasetToBrokerDataset((Dataset) target)); + return (RT) new RelatedDataset(sourceId, relType, + ConversionUtils.oafDatasetToBrokerDataset((Dataset) target)); } else if (clazz == RelatedPublication.class) { - return (RT) new RelatedPublication(sourceId, relType, ConversionUtils.oafPublicationToBrokerPublication((Publication) target)); + return (RT) new RelatedPublication(sourceId, relType, + ConversionUtils.oafPublicationToBrokerPublication((Publication) target)); } else { return null; } From 1bc1d15eaf1508bdd3bc5a3a5b3aa48d57f63708 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 16 Jun 2020 16:54:28 +0200 Subject: [PATCH 85/88] stubbing for mock datasource.identities must be typed as array --- .../dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json index f4c5f97ed..0f1da7095 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json @@ -6,9 +6,10 @@ }, { "field": "identities", - "type": "not_used", + "type": "array", "value": [ "274269ac6f3b::2579-5449", + "piwik:13", null ] }, From 306669209fa58206635ab094f2f3ef28963b34f1 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 16 Jun 2020 16:54:44 +0200 Subject: [PATCH 86/88] code formatting --- .../eu/dnetlib/dhp/schema/oaf/Measure.java | 79 ++++++++++--------- .../eu/dnetlib/dhp/schema/oaf/Programme.java | 1 - .../eu/dnetlib/dhp/schema/oaf/Relation.java | 14 ++-- .../eu/dnetlib/dhp/schema/oaf/Result.java | 2 +- .../dnetlib/dhp/schema/oaf/MeasureTest.java | 67 +++++++++------- .../actionmanager/project/ProjectSubset.java | 15 ++-- .../project/SparkUpdateProjectTest.java | 1 - 7 files changed, 96 insertions(+), 83 deletions(-) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java index c37e76061..c0c14d10d 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java @@ -1,56 +1,59 @@ + package eu.dnetlib.dhp.schema.oaf; -import com.google.common.base.Objects; - import java.util.List; +import com.google.common.base.Objects; + /** * Represent a measure, must be further described by a system available resource providing name and descriptions. */ public class Measure { - /** - * Unique measure identifier. - */ - private String id; + /** + * Unique measure identifier. + */ + private String id; - /** - * List of units associated with this measure. KeyValue provides a pair to store the laber (key) and the value, - * plus common provenance information. - */ - private List unit; + /** + * List of units associated with this measure. KeyValue provides a pair to store the laber (key) and the value, plus + * common provenance information. + */ + private List unit; - public String getId() { - return id; - } + public String getId() { + return id; + } - public void setId(String id) { - this.id = id; - } + public void setId(String id) { + this.id = id; + } - public List getUnit() { - return unit; - } + public List getUnit() { + return unit; + } - public void setUnit(List unit) { - this.unit = unit; - } + public void setUnit(List unit) { + this.unit = unit; + } - public void mergeFrom(Measure m) { - //TODO - } + public void mergeFrom(Measure m) { + // TODO + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Measure measure = (Measure) o; - return Objects.equal(id, measure.id) && - Objects.equal(unit, measure.unit); - } + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Measure measure = (Measure) o; + return Objects.equal(id, measure.id) && + Objects.equal(unit, measure.unit); + } - @Override - public int hashCode() { - return Objects.hashCode(id, unit); - } + @Override + public int hashCode() { + return Objects.hashCode(id, unit); + } } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java index c5259d07e..00dc32fbc 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java @@ -35,5 +35,4 @@ public class Programme implements Serializable { return Objects.equals(code, programme.code); } - } diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java index d77bd7d73..17a50d7ac 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java @@ -8,10 +8,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * Relation models any edge between two nodes in the OpenAIRE graph. It has a source id and a target id - * pointing to graph node identifiers and it is further characterised by the semantic of the link through the fields - * relType, subRelType and relClass. Provenance information is modeled according to the dataInfo element and collectedFrom, - * while individual relationship types can provide extra information via the properties field. + * Relation models any edge between two nodes in the OpenAIRE graph. It has a source id and a target id pointing to + * graph node identifiers and it is further characterised by the semantic of the link through the fields relType, + * subRelType and relClass. Provenance information is modeled according to the dataInfo element and collectedFrom, while + * individual relationship types can provide extra information via the properties field. */ public class Relation extends Oaf { @@ -26,7 +26,8 @@ public class Relation extends Oaf { private String subRelType; /** - * Indicates the direction of the relationship, values include 'isSupplementTo', 'isSupplementedBy', 'merges, 'isMergedIn'. + * Indicates the direction of the relationship, values include 'isSupplementTo', 'isSupplementedBy', 'merges, + * 'isMergedIn'. */ private String relClass; @@ -51,7 +52,8 @@ public class Relation extends Oaf { private String validationDate; /** - * List of relation specific properties. Values include 'similarityLevel', indicating the similarity score between a pair of publications. + * List of relation specific properties. Values include 'similarityLevel', indicating the similarity score between a + * pair of publications. */ private List properties = new ArrayList<>(); diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java index 44737d46e..0dc76fd43 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java @@ -241,7 +241,7 @@ public class Result extends OafEntity implements Serializable { Result r = (Result) e; - //TODO consider merging also Measures + // TODO consider merging also Measures instance = mergeLists(instance, r.getInstance()); diff --git a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java index 4275e2c56..26b4407c9 100644 --- a/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java +++ b/dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java @@ -1,48 +1,57 @@ + package eu.dnetlib.dhp.schema.oaf; +import java.io.IOException; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.List; public class MeasureTest { - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() - .setSerializationInclusion(JsonInclude.Include.NON_NULL); + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL); - @Test - public void testMeasureSerialization() throws IOException { + @Test + public void testMeasureSerialization() throws IOException { - Measure popularity = new Measure(); - popularity.setId("popularity"); - popularity.setUnit(Lists.newArrayList( - unit("score", "0.5"))); + Measure popularity = new Measure(); + popularity.setId("popularity"); + popularity + .setUnit( + Lists + .newArrayList( + unit("score", "0.5"))); - Measure influence = new Measure(); - influence.setId("influence"); - influence.setUnit(Lists.newArrayList( - unit("score", "0.3"))); + Measure influence = new Measure(); + influence.setId("influence"); + influence + .setUnit( + Lists + .newArrayList( + unit("score", "0.3"))); - List m = Lists.newArrayList(popularity, influence); + List m = Lists.newArrayList(popularity, influence); - String s = OBJECT_MAPPER.writeValueAsString(m); - System.out.println(s); + String s = OBJECT_MAPPER.writeValueAsString(m); + System.out.println(s); - List mm = OBJECT_MAPPER.readValue(s, new TypeReference>() { }); + List mm = OBJECT_MAPPER.readValue(s, new TypeReference>() { + }); - Assertions.assertNotNull(mm); - } + Assertions.assertNotNull(mm); + } - private KeyValue unit(String key, String value) { - KeyValue unit = new KeyValue(); - unit.setKey(key); - unit.setValue(value); - return unit; - } + private KeyValue unit(String key, String value) { + KeyValue unit = new KeyValue(); + unit.setKey(key); + unit.setValue(value); + return unit; + } } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java index cfbb62f21..2fccbc516 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java @@ -1,16 +1,17 @@ + package eu.dnetlib.dhp.actionmanager.project; import java.io.Serializable; public class ProjectSubset implements Serializable { - private String code; + private String code; - public String getCode() { - return code; - } + public String getCode() { + return code; + } - public void setCode(String code) { - this.code = code; - } + public void setCode(String code) { + this.code = code; + } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java index 4d3ec140b..718cd8ebe 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/SparkUpdateProjectTest.java @@ -90,6 +90,5 @@ public class SparkUpdateProjectTest { Assertions.assertEquals(14, tmp.count()); - } } From 64f02de5d3765f741cdb458e42c8a20a725aaf4a Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 16 Jun 2020 17:48:51 +0200 Subject: [PATCH 87/88] updated workflow definition to include the cleaning step --- .../eu/dnetlib/dhp/wf/profiles/provision.xml | 92 +++++++++++++++---- 1 file changed, 73 insertions(+), 19 deletions(-) diff --git a/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/provision.xml b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/provision.xml index 487afee4f..28cbde70d 100644 --- a/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/provision.xml +++ b/dhp-workflows/dhp-worfklow-profiles/src/main/resources/eu/dnetlib/dhp/wf/profiles/provision.xml @@ -4,7 +4,7 @@ - + Data Provision [OCEAN] @@ -61,11 +61,23 @@ + + + Set the target path to store the CLEANED graph + + cleanedGraphPath + /tmp/beta_provision/graph/05_graph_cleaned + + + + + + Set the target path to store the ORCID enriched graph orcidGraphPath - /tmp/beta_provision/graph/05_graph_orcid + /tmp/beta_provision/graph/06_graph_orcid @@ -75,7 +87,7 @@ Set the target path to store the BULK TAGGED graph bulkTaggingGraphPath - /tmp/beta_provision/graph/06_graph_bulktagging + /tmp/beta_provision/graph/07_graph_bulktagging @@ -85,7 +97,7 @@ Set the target path to store the AFFILIATION from INSTITUTIONAL REPOS graph affiliationGraphPath - /tmp/beta_provision/graph/07_graph_affiliation + /tmp/beta_provision/graph/08_graph_affiliation @@ -95,7 +107,7 @@ Set the target path to store the COMMUNITY from SELECTED SOURCES graph communityOrganizationGraphPath - /tmp/beta_provision/graph/08_graph_comunity_organization + /tmp/beta_provision/graph/09_graph_comunity_organization @@ -105,7 +117,7 @@ Set the target path to store the FUNDING from SEMANTIC RELATION graph fundingGraphPath - /tmp/beta_provision/graph/09_graph_funding + /tmp/beta_provision/graph/10_graph_funding @@ -115,7 +127,7 @@ Set the target path to store the COMMUNITY from SEMANTIC RELATION graph communitySemRelGraphPath - /tmp/beta_provision/graph/10_graph_comunity_sem_rel + /tmp/beta_provision/graph/11_graph_comunity_sem_rel @@ -125,7 +137,7 @@ Set the target path to store the COUNTRY enriched graph countryGraphPath - /tmp/beta_provision/graph/11_graph_country + /tmp/beta_provision/graph/12_graph_country @@ -135,7 +147,7 @@ Set the target path to store the blacklisted graph blacklistedGraphPath - /tmp/beta_provision/graph/12_graph_blacklisted + /tmp/beta_provision/graph/13_graph_blacklisted @@ -217,7 +229,8 @@ IIS { - 'graphOutputPath' : 'aggregatorGraphPath' + 'graphOutputPath' : 'aggregatorGraphPath', + 'isLookupUrl' : 'isLookUpUrl' } @@ -227,8 +240,8 @@ 'mongoDb' : 'mdstore', 'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus', 'postgresUser' : 'dnet', - 'postgresPassword' : 'dnetPwd', - 'reuseContent' : 'true', + 'postgresPassword' : '', + 'reuseContent' : 'false', 'contentPath' : '/tmp/beta_provision/aggregator', 'workingDir' : '/tmp/beta_provision/working_dir/aggregator' } @@ -255,6 +268,8 @@ { 'oozie.wf.application.path' : '/lib/dnet/actionmanager/wf/main/oozie_app', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', 'activePromoteDatasetActionPayload' : 'true', 'activePromoteDatasourceActionPayload' : 'true', 'activePromoteOrganizationActionPayload' : 'true', @@ -315,6 +330,8 @@ { 'oozie.wf.application.path' : '/lib/dnet/actionmanager/wf/main/oozie_app', + 'sparkExecutorCores' : '3', + 'sparkExecutorMemory' : '10G', 'activePromoteDatasetActionPayload' : 'true', 'activePromoteDatasourceActionPayload' : 'true', 'activePromoteOrganizationActionPayload' : 'true', @@ -354,7 +371,43 @@ build-report - + + + + + + clean the properties in the graph typed as Qualifier according to the vocabulary indicated in schemeid + + executeOozieJob + IIS + + { + 'graphInputPath' : 'consistentGraphPath', + 'graphOutputPath': 'cleanedGraphPath', + 'isLookupUrl': 'isLookUpUrl' + } + + + { + 'oozie.wf.application.path' : '/lib/dnet/oa/graph/clean/oozie_app', + 'workingPath' : '/tmp/beta_provision/working_dir/clean' + } + + build-report + + + + + + + + Do we skip the graph enrichment steps? (Yes to prepare the graph for the IIS) + + NO + + + + @@ -364,7 +417,7 @@ IIS { - 'sourcePath' : 'consistentGraphPath', + 'sourcePath' : 'cleanedGraphPath', 'outputPath': 'orcidGraphPath' } @@ -552,7 +605,7 @@ 'workingDir' : '/tmp/beta_provision/working_dir/blacklist', 'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus', 'postgresUser' : 'dnet', - 'postgresPassword' : 'dnetPwd' + 'postgresPassword' : '' } build-report @@ -561,12 +614,13 @@ + - wf_20200509_100941_857 - 2020-05-09T13:26:09+00:00 - FAILURE - eu.dnetlib.data.hadoop.rmi.HadoopServiceException: hadoop job: 0002933-200403132837156-oozie-oozi-W failed with status: KILLED, oozie log: 2020-05-09 13:23:31,194 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[] No results found 2020-05-09 13:23:31,216 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] Start action [0002933-200403132837156-oozie-oozi-W@:start:] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:31,216 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] [***0002933-200403132837156-oozie-oozi-W@:start:***]Action status=DONE 2020-05-09 13:23:31,216 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] [***0002933-200403132837156-oozie-oozi-W@:start:***]Action updated in DB! 2020-05-09 13:23:31,257 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] No results found 2020-05-09 13:23:31,275 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@:start:] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@:start: 2020-05-09 13:23:31,275 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W 2020-05-09 13:23:31,314 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] Start action [0002933-200403132837156-oozie-oozi-W@reset-outputpath] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:33,897 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] [***0002933-200403132837156-oozie-oozi-W@reset-outputpath***]Action status=DONE 2020-05-09 13:23:33,897 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] [***0002933-200403132837156-oozie-oozi-W@reset-outputpath***]Action updated in DB! 2020-05-09 13:23:33,947 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] No results found 2020-05-09 13:23:33,966 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] Start action [0002933-200403132837156-oozie-oozi-W@copy_entities] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:33,966 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] [***0002933-200403132837156-oozie-oozi-W@copy_entities***]Action status=DONE 2020-05-09 13:23:33,966 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] [***0002933-200403132837156-oozie-oozi-W@copy_entities***]Action updated in DB! 2020-05-09 13:23:34,012 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No results found 2020-05-09 13:23:34,018 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No results found 2020-05-09 13:23:34,023 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No results found 2020-05-09 13:23:34,029 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No results found 2020-05-09 13:23:34,124 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] Start action [0002933-200403132837156-oozie-oozi-W@copy_relation] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:34,130 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] Start action [0002933-200403132837156-oozie-oozi-W@copy_projects] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:34,130 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] Start action [0002933-200403132837156-oozie-oozi-W@copy_datasources] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:34,140 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] Start action [0002933-200403132837156-oozie-oozi-W@copy_organization] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:23:35,010 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] checking action, hadoop job ID [job_1585920557248_14569] status [RUNNING] 2020-05-09 13:23:35,018 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] [***0002933-200403132837156-oozie-oozi-W@copy_projects***]Action status=RUNNING 2020-05-09 13:23:35,018 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] [***0002933-200403132837156-oozie-oozi-W@copy_projects***]Action updated in DB! 2020-05-09 13:23:35,022 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] checking action, hadoop job ID [job_1585920557248_14568] status [RUNNING] 2020-05-09 13:23:35,027 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_projects 2020-05-09 13:23:35,028 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] [***0002933-200403132837156-oozie-oozi-W@copy_relation***]Action status=RUNNING 2020-05-09 13:23:35,028 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] [***0002933-200403132837156-oozie-oozi-W@copy_relation***]Action updated in DB! 2020-05-09 13:23:35,031 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] checking action, hadoop job ID [job_1585920557248_14570] status [RUNNING] 2020-05-09 13:23:35,035 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] [***0002933-200403132837156-oozie-oozi-W@copy_datasources***]Action status=RUNNING 2020-05-09 13:23:35,035 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] [***0002933-200403132837156-oozie-oozi-W@copy_datasources***]Action updated in DB! 2020-05-09 13:23:35,037 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_relation 2020-05-09 13:23:35,048 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_datasources 2020-05-09 13:23:35,072 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] checking action, hadoop job ID [job_1585920557248_14571] status [RUNNING] 2020-05-09 13:23:35,076 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] [***0002933-200403132837156-oozie-oozi-W@copy_organization***]Action status=RUNNING 2020-05-09 13:23:35,076 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] [***0002933-200403132837156-oozie-oozi-W@copy_organization***]Action updated in DB! 2020-05-09 13:23:35,084 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_organization 2020-05-09 13:23:35,090 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_entities] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_entities 2020-05-09 13:23:35,090 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@reset-outputpath] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@reset-outputpath 2020-05-09 13:23:58,926 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] callback for action [0002933-200403132837156-oozie-oozi-W@copy_datasources] 2020-05-09 13:23:59,085 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] checking action, hadoop job ID [job_1585920557248_14570] status [RUNNING] 2020-05-09 13:23:59,242 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] callback for action [0002933-200403132837156-oozie-oozi-W@copy_projects] 2020-05-09 13:23:59,386 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] checking action, hadoop job ID [job_1585920557248_14569] status [RUNNING] 2020-05-09 13:24:01,343 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] callback for action [0002933-200403132837156-oozie-oozi-W@copy_datasources] 2020-05-09 13:24:01,418 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] Hadoop Jobs launched : [job_1585920557248_14573] 2020-05-09 13:24:01,418 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] action completed, external ID [job_1585920557248_14570] 2020-05-09 13:24:01,493 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_datasources] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_datasources 2020-05-09 13:24:01,935 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] callback for action [0002933-200403132837156-oozie-oozi-W@copy_projects] 2020-05-09 13:24:02,012 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] Hadoop Jobs launched : [job_1585920557248_14572] 2020-05-09 13:24:02,012 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] action completed, external ID [job_1585920557248_14569] 2020-05-09 13:24:02,076 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_projects] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_projects 2020-05-09 13:25:03,172 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] callback for action [0002933-200403132837156-oozie-oozi-W@copy_organization] 2020-05-09 13:25:03,336 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] checking action, hadoop job ID [job_1585920557248_14571] status [RUNNING] 2020-05-09 13:25:05,598 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] callback for action [0002933-200403132837156-oozie-oozi-W@copy_organization] 2020-05-09 13:25:05,688 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] Hadoop Jobs launched : [job_1585920557248_14574] 2020-05-09 13:25:05,691 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] action completed, external ID [job_1585920557248_14571] 2020-05-09 13:25:05,748 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_organization] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_organization 2020-05-09 13:25:23,274 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] callback for action [0002933-200403132837156-oozie-oozi-W@copy_relation] 2020-05-09 13:25:23,409 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] checking action, hadoop job ID [job_1585920557248_14568] status [RUNNING] 2020-05-09 13:25:25,419 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] callback for action [0002933-200403132837156-oozie-oozi-W@copy_relation] 2020-05-09 13:25:25,510 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] Hadoop Jobs launched : [job_1585920557248_14575] 2020-05-09 13:25:25,511 INFO org.apache.oozie.action.hadoop.DistcpActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] action completed, external ID [job_1585920557248_14568] 2020-05-09 13:25:25,565 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] No results found 2020-05-09 13:25:25,585 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] Start action [0002933-200403132837156-oozie-oozi-W@copy_wait] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,585 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] [***0002933-200403132837156-oozie-oozi-W@copy_wait***]Action status=DONE 2020-05-09 13:25:25,585 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] [***0002933-200403132837156-oozie-oozi-W@copy_wait***]Action updated in DB! 2020-05-09 13:25:25,627 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] No results found 2020-05-09 13:25:25,648 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] Start action [0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,648 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] [***0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1***]Action status=DONE 2020-05-09 13:25:25,648 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] [***0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1***]Action updated in DB! 2020-05-09 13:25:25,694 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No results found 2020-05-09 13:25:25,700 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No results found 2020-05-09 13:25:25,706 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No results found 2020-05-09 13:25:25,711 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No results found 2020-05-09 13:25:25,801 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] Start action [0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,825 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] Start action [0002933-200403132837156-oozie-oozi-W@join_prepare_software] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,825 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] Start action [0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:25,828 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] Start action [0002933-200403132837156-oozie-oozi-W@join_prepare_publication] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:27,165 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] checking action, hadoop job ID [job_1585920557248_14578] status [RUNNING] 2020-05-09 13:25:27,170 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] [***0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct***]Action status=RUNNING 2020-05-09 13:25:27,170 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] [***0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct***]Action updated in DB! 2020-05-09 13:25:27,179 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] checking action, hadoop job ID [job_1585920557248_14577] status [RUNNING] 2020-05-09 13:25:27,181 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct 2020-05-09 13:25:27,183 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] [***0002933-200403132837156-oozie-oozi-W@join_prepare_software***]Action status=RUNNING 2020-05-09 13:25:27,183 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] [***0002933-200403132837156-oozie-oozi-W@join_prepare_software***]Action updated in DB! 2020-05-09 13:25:27,188 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_software 2020-05-09 13:25:27,617 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] checking action, hadoop job ID [job_1585920557248_14576] status [RUNNING] 2020-05-09 13:25:27,622 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] [***0002933-200403132837156-oozie-oozi-W@join_prepare_publication***]Action status=RUNNING 2020-05-09 13:25:27,622 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] [***0002933-200403132837156-oozie-oozi-W@join_prepare_publication***]Action updated in DB! 2020-05-09 13:25:27,625 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] checking action, hadoop job ID [job_1585920557248_14579] status [RUNNING] 2020-05-09 13:25:27,628 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_publication 2020-05-09 13:25:27,629 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] [***0002933-200403132837156-oozie-oozi-W@join_prepare_dataset***]Action status=RUNNING 2020-05-09 13:25:27,629 INFO org.apache.oozie.command.wf.ForkedActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] [***0002933-200403132837156-oozie-oozi-W@join_prepare_dataset***]Action updated in DB! 2020-05-09 13:25:27,634 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_dataset 2020-05-09 13:25:27,639 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@fork_prepare_assoc_step1 2020-05-09 13:25:27,639 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_wait] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_wait 2020-05-09 13:25:27,640 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@copy_relation] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@copy_relation 2020-05-09 13:25:41,416 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] callback for action [0002933-200403132837156-oozie-oozi-W@join_prepare_software] 2020-05-09 13:25:41,490 INFO org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] action completed, external ID [job_1585920557248_14577] 2020-05-09 13:25:41,495 WARN org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] Launcher ERROR, reason: Main class [org.apache.oozie.action.hadoop.SparkMain], main() threw exception, File file:/data/3/yarn/nm/usercache/dnet.beta/appcache/application_1585920557248_14577/container_e68_1585920557248_14577_01_000002/dhp-propagation-1.1.8-SNAPSHOT.jar does not exist 2020-05-09 13:25:41,495 WARN org.apache.oozie.action.hadoop.SparkActionExecutor: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] Launcher exception: File file:/data/3/yarn/nm/usercache/dnet.beta/appcache/application_1585920557248_14577/container_e68_1585920557248_14577_01_000002/dhp-propagation-1.1.8-SNAPSHOT.jar does not exist java.io.FileNotFoundException: File file:/data/3/yarn/nm/usercache/dnet.beta/appcache/application_1585920557248_14577/container_e68_1585920557248_14577_01_000002/dhp-propagation-1.1.8-SNAPSHOT.jar does not exist at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:598) at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:811) at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:588) at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:432) at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:340) at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:292) at org.apache.spark.deploy.yarn.Client.copyFileToRemote(Client.scala:404) at org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:496) at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$9.apply(Client.scala:595) at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$9.apply(Client.scala:594) at scala.Option.foreach(Option.scala:257) at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:594) at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:886) at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:180) at org.apache.spark.deploy.yarn.Client.run(Client.scala:1156) at org.apache.spark.deploy.yarn.YarnClusterApplication.start(Client.scala:1608) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:849) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) at org.apache.oozie.action.hadoop.SparkMain.runSpark(SparkMain.java:178) at org.apache.oozie.action.hadoop.SparkMain.run(SparkMain.java:90) at org.apache.oozie.action.hadoop.LauncherMain.run(LauncherMain.java:81) at org.apache.oozie.action.hadoop.SparkMain.main(SparkMain.java:57) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.oozie.action.hadoop.LauncherMapper.map(LauncherMapper.java:235) at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:54) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:459) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) 2020-05-09 13:25:41,514 INFO org.apache.oozie.command.wf.ActionEndXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] ERROR is considered as FAILED for SLA 2020-05-09 13:25:41,541 INFO org.apache.oozie.service.JPAService: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] No results found 2020-05-09 13:25:41,580 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] Start action [0002933-200403132837156-oozie-oozi-W@Kill] with user-retry state : userRetryCount [0], userRetryMax [0], userRetryInterval [10] 2020-05-09 13:25:41,580 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] [***0002933-200403132837156-oozie-oozi-W@Kill***]Action status=DONE 2020-05-09 13:25:41,580 INFO org.apache.oozie.command.wf.ActionStartXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] [***0002933-200403132837156-oozie-oozi-W@Kill***]Action updated in DB! 2020-05-09 13:25:41,692 WARN org.apache.oozie.workflow.lite.LiteWorkflowInstance: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] Workflow completed [KILLED], killing [3] running nodes 2020-05-09 13:25:41,760 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@Kill] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@Kill 2020-05-09 13:25:41,766 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_software] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_software 2020-05-09 13:25:41,852 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct 2020-05-09 13:25:41,914 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] callback for action [0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] 2020-05-09 13:25:41,920 ERROR org.apache.oozie.command.wf.CompletedActionXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] XException, org.apache.oozie.command.CommandException: E0800: Action it is not running its in [KILLED] state, action [0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] at org.apache.oozie.command.wf.CompletedActionXCommand.eagerVerifyPrecondition(CompletedActionXCommand.java:92) at org.apache.oozie.command.XCommand.call(XCommand.java:257) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.oozie.service.CallableQueueService$CallableWrapper.run(CallableQueueService.java:179) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2020-05-09 13:25:41,938 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_publication 2020-05-09 13:25:42,005 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] callback for action [0002933-200403132837156-oozie-oozi-W@join_prepare_publication] 2020-05-09 13:25:42,010 ERROR org.apache.oozie.command.wf.CompletedActionXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_publication] XException, org.apache.oozie.command.CommandException: E0800: Action it is not running its in [KILLED] state, action [0002933-200403132837156-oozie-oozi-W@join_prepare_publication] at org.apache.oozie.command.wf.CompletedActionXCommand.eagerVerifyPrecondition(CompletedActionXCommand.java:92) at org.apache.oozie.command.XCommand.call(XCommand.java:257) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.oozie.service.CallableQueueService$CallableWrapper.run(CallableQueueService.java:179) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2020-05-09 13:25:42,028 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[dnet.beta] GROUP[-] TOKEN[] APP[orcid_to_result_from_semrel_propagation] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W 2020-05-09 13:25:42,028 INFO org.apache.oozie.command.wf.WorkflowNotificationXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_dataset] No Notification URL is defined. Therefore nothing to notify for job 0002933-200403132837156-oozie-oozi-W@join_prepare_dataset 2020-05-09 13:25:42,113 INFO org.apache.oozie.servlet.CallbackServlet: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] callback for action [0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] 2020-05-09 13:25:42,116 ERROR org.apache.oozie.command.wf.CompletedActionXCommand: SERVER[iis-cdh5-test-m3.ocean.icm.edu.pl] USER[-] GROUP[-] TOKEN[] APP[-] JOB[0002933-200403132837156-oozie-oozi-W] ACTION[0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] XException, org.apache.oozie.command.CommandException: E0800: Action it is not running its in [KILLED] state, action [0002933-200403132837156-oozie-oozi-W@join_prepare_otherresearchproduct] at org.apache.oozie.command.wf.CompletedActionXCommand.eagerVerifyPrecondition(CompletedActionXCommand.java:92) at org.apache.oozie.command.XCommand.call(XCommand.java:257) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.oozie.service.CallableQueueService$CallableWrapper.run(CallableQueueService.java:179) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) + wf_20200615_163630_609 + 2020-06-15T17:08:00+00:00 + SUCCESS + \ No newline at end of file From 11b77b9f4e29a0d09cab1f1a3ab49419c6735517 Mon Sep 17 00:00:00 2001 From: miconis Date: Tue, 16 Jun 2020 18:31:11 +0200 Subject: [PATCH 88/88] json dumps for entity merge test modified to fit the new model. title merge adjusted to fix the error --- .../src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java | 4 ++-- .../eu/dnetlib/dhp/dedup/json/publication_merge.json | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java index c9d0ac7c7..82e8fd05d 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java @@ -259,9 +259,9 @@ public class Result extends OafEntity implements Serializable { StructuredProperty newMainTitle = null; if (r.getTitle() != null) { newMainTitle = getMainTitle(r.getTitle()); - if (newMainTitle != null && title != null) { + if (newMainTitle != null) { final StructuredProperty p = newMainTitle; - title = title.stream().filter(t -> t != p).collect(Collectors.toList()); + r.setTitle(r.getTitle().stream().filter(t -> t != p).collect(Collectors.toList())); } } diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/publication_merge.json b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/publication_merge.json index 28548c532..ae688e746 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/publication_merge.json +++ b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/publication_merge.json @@ -1,3 +1,3 @@ -{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": false, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.95"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}], "id": "50|a89337edbe55::4930db9e954866d70916cbfba9f81f97", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": [], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-9999"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2019-11-05T14:49:22.351Z", "fulltext": [], "dateoftransformation": "2019-11-05T16:10:58.988Z", "description": [], "format": [], "journal": {"issnPrinted": "1459-6067", "conferencedate": "", "conferenceplace": "", "name": "Agricultural and Food Science", "edition": "", "iss": "3", "sp": "", "vol": "27", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "1795-1895", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "eng", "classname": "English", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [], "extraInfo": [], "originalId": [], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2018-09-30"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} -{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "sysimport:crosswalk:repository", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "doi", "classname": "doi", "schemename": "dnet:pid_types", "schemeid": "dnet:pid_types"}, "value": "10.1016/j.nicl.2015.11.006"}], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}], "id": "50|base_oa_____::0968af610a356656706657e4f234b340", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "NeuroImage: Clinical", "key": "10|doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "http://creativecommons.org/licenses/by-nc-nd/4.0/"}, "url": ["http://dx.doi.org/10.1016/j.nicl.2015.11.006"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:publication_resource", "schemeid": "dnet:publication_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}, {"surname": "Klein", "name": "Christine", "pid": [], "rank": 10, "affiliation": [], "fullname": "Klein, Christine"}, {"surname": "Deuschl", "name": "Gu\\u0308nther", "pid": [], "rank": 11, "affiliation": [], "fullname": "Deuschl, G\\u00fcnther"}, {"surname": "Eimeren", "name": "Thilo", "pid": [], "rank": 12, "affiliation": [], "fullname": "van Eimeren, Thilo"}, {"surname": "Witt", "name": "Karsten", "pid": [], "rank": 13, "affiliation": [], "fullname": "Witt, Karsten"}], "source": [], "dateofcollection": "2017-07-27T19:04:09.131Z", "fulltext": [], "dateoftransformation": "2019-01-23T10:15:19.582Z", "description": [], "format": [], "journal": {"issnPrinted": "2213-1582", "conferencedate": "", "conferenceplace": "", "name": "NeuroImage: Clinical", "edition": "", "iss": "", "sp": "63", "vol": "10", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "70", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Elsevier BV"}, "language": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "IT", "classname": "Italy", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["10.1016/j.nicl.2015.11.006"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} -{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}], "id": "50|CrisUnsNoviS::9f9d014eea45dab432cab636c4c9cf39", "subject": [], "instance": [{"refereed": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": ["https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2019-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "accessright": {"classid": "UNKNOWN", "classname": "UNKNOWN", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}, {"qualifier": {"classid": "pubmed", "classname": "pubmed"}, "value": "pubmed.it"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [{"qualifier": {"classid": "id", "classname": "id"}, "value": "12345678"}], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-1023"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2020-03-10T15:05:38.685Z", "fulltext": [], "dateoftransformation": "2020-03-11T20:11:13.15Z", "description": [], "format": [], "journal": {"issnPrinted": "", "conferencedate": "", "conferenceplace": "", "name": "", "edition": "", "iss": "", "sp": "", "vol": "", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "en", "classname": "en", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "UNKNOWN", "classname": "not available", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "FI", "classname": "Finland", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["(BISIS)113444", "https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "test title", "classname": "test title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Antichains of copies of ultrahomogeneous structures"}]} \ No newline at end of file +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": false, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.95"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}], "id": "50|a89337edbe55::4930db9e954866d70916cbfba9f81f97", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": [], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Journal.fi", "key": "10|openaire____::6eef8049d0feedc089ee009abca55e35"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-9999"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2019-11-05T14:49:22.351Z", "fulltext": [], "dateoftransformation": "2019-11-05T16:10:58.988Z", "description": [], "format": [], "journal": {"issnPrinted": "1459-6067", "conferencedate": "", "conferenceplace": "", "name": "Agricultural and Food Science", "edition": "", "iss": "3", "sp": "", "vol": "27", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "1795-1895", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "eng", "classname": "English", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [], "extraInfo": [], "originalId": [], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2018-09-30"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "sysimport:crosswalk:repository", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "pid": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "doi", "classname": "doi", "schemename": "dnet:pid_types", "schemeid": "dnet:pid_types"}, "value": "10.1016/j.nicl.2015.11.006"}], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}], "id": "50|base_oa_____::0968af610a356656706657e4f234b340", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "NeuroImage: Clinical", "key": "10|doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "http://creativecommons.org/licenses/by-nc-nd/4.0/"}, "url": ["http://dx.doi.org/10.1016/j.nicl.2015.11.006"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "BASE (Open Access Aggregator)", "key": "10|openaire____::df45502607927471ecf8a6ae83683ff5"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemename": "dnet:publication_resource", "schemeid": "dnet:publication_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}, {"surname": "Klein", "name": "Christine", "pid": [], "rank": 10, "affiliation": [], "fullname": "Klein, Christine"}, {"surname": "Deuschl", "name": "Gu\\u0308nther", "pid": [], "rank": 11, "affiliation": [], "fullname": "Deuschl, G\\u00fcnther"}, {"surname": "Eimeren", "name": "Thilo", "pid": [], "rank": 12, "affiliation": [], "fullname": "van Eimeren, Thilo"}, {"surname": "Witt", "name": "Karsten", "pid": [], "rank": 13, "affiliation": [], "fullname": "Witt, Karsten"}], "source": [], "dateofcollection": "2017-07-27T19:04:09.131Z", "fulltext": [], "dateoftransformation": "2019-01-23T10:15:19.582Z", "description": [], "format": [], "journal": {"issnPrinted": "2213-1582", "conferencedate": "", "conferenceplace": "", "name": "NeuroImage: Clinical", "edition": "", "iss": "", "sp": "63", "vol": "10", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "70", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "Elsevier BV"}, "language": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "IT", "classname": "Italy", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["10.1016/j.nicl.2015.11.006"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "main title", "classname": "main title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Altered brain activation in a reversal learning task unmasks adaptive changes in cognitive control in writer's cramp"}]} +{"context": [], "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "sysimport:crosswalk:datasetarchive", "classname": "sysimport:crosswalk:datasetarchive", "schemename": "dnet:provenanceActions", "schemeid": "dnet:provenanceActions"}, "inferred": true, "inferenceprovenance": "dedup-similarity-result-levenstein", "invisible": false, "trust": "0.9"}, "resourcetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}, "pid": [], "contributor": [], "resulttype": {"classid": "publication", "classname": "publication", "schemename": "dnet:result_typologies", "schemeid": "dnet:result_typologies"}, "relevantdate": [], "collectedfrom": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}], "id": "50|CrisUnsNoviS::9f9d014eea45dab432cab636c4c9cf39", "subject": [], "instance": [{"refereed": null, "hostedby": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "processingchargeamount": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "license": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "url": ["https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "distributionlocation": "", "processingchargecurrency": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2019-01-01"}, "collectedfrom": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "CRIS UNS (Current Research Information System University of Novi Sad)", "key": "10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556"}, "accessright": {"classid": "UNKNOWN", "classname": "UNKNOWN", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "instancetype": {"classid": "0004", "classname": "Conference object", "schemename": "dnet:dataCite_resource", "schemeid": "dnet:dataCite_resource"}}], "embargoenddate": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "lastupdatetimestamp": 0, "author": [{"surname": "Zeuner", "name": "Kirsten E.", "pid": [], "rank": 1, "affiliation": [], "fullname": "Zeuner, Kirsten E."}, {"surname": "Knutzen", "name": "Arne", "pid": [], "rank": 2, "affiliation": [], "fullname": "Knutzen, Arne"}, {"surname": "Granert", "name": "Oliver", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0002-0656-1023"}, {"qualifier": {"classid": "pubmed", "classname": "pubmed"}, "value": "pubmed.it"}], "rank": 3, "affiliation": [], "fullname": "Granert, Oliver"}, {"surname": "Sablowsky", "name": "Simone", "pid": [{"qualifier": {"classid": "id", "classname": "id"}, "value": "12345678"}], "rank": 4, "affiliation": [], "fullname": "Sablowsky, Simone"}, {"surname": "Go\\u0308tz", "name": "Julia", "pid": [], "rank": 5, "affiliation": [], "fullname": "G\\u00f6tz, Julia"}, {"surname": "Wolff", "name": "Stephan", "pid": [], "rank": 6, "affiliation": [], "fullname": "Wolff, Stephan"}, {"surname": "Jansen", "name": "Olav", "pid": [{"qualifier": {"classid": "ORCID", "classname": "ORCID"}, "value": "0000-0000-0656-1023"},{"qualifier": {"classid": "id", "classname": "id"}, "value": "987654321"}], "rank": 7, "affiliation": [], "fullname": "Jansen, Olav"}, {"surname": "Dressler", "name": "Dirk", "pid": [], "rank": 8, "affiliation": [], "fullname": "Dressler, Dirk"}, {"surname": "Schneider", "name": "Susanne A.", "pid": [], "rank": 9, "affiliation": [], "fullname": "Schneider, Susanne A."}], "source": [], "dateofcollection": "2020-03-10T15:05:38.685Z", "fulltext": [], "dateoftransformation": "2020-03-11T20:11:13.15Z", "description": [], "format": [], "journal": {"issnPrinted": "", "conferencedate": "", "conferenceplace": "", "name": "", "edition": "", "iss": "", "sp": "", "vol": "", "dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "issnOnline": "", "ep": "", "issnLinking": ""}, "coverage": [], "publisher": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": ""}, "language": {"classid": "en", "classname": "en", "schemename": "dnet:languages", "schemeid": "dnet:languages"}, "bestaccessright": {"classid": "UNKNOWN", "classname": "not available", "schemename": "dnet:access_modes", "schemeid": "dnet:access_modes"}, "country": [{"classid": "FI", "classname": "Finland", "schemeid": "dnet:countries", "schemename": "dnet:countries"}], "extraInfo": [], "originalId": ["(BISIS)113444", "https://www.cris.uns.ac.rs/record.jsf?recordId=113444&source=OpenAIRE&language=en"], "dateofacceptance": {"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "value": "2016-01-01"}, "title": [{"dataInfo": {"deletedbyinference": false, "provenanceaction": {"classid": "", "classname": "", "schemename": "", "schemeid": ""}, "inferred": false, "inferenceprovenance": "", "invisible": false, "trust": ""}, "qualifier": {"classid": "test title", "classname": "test title", "schemename": "dnet:dataCite_title", "schemeid": "dnet:dataCite_title"}, "value": "Antichains of copies of ultrahomogeneous structures"}]} \ No newline at end of file