WIP: promote job tests and test resources implementation snapshot

dhp-workflows/dhp-actionmanager/src/test/java/eu/dnetlib/dhp/actionmanager/PromoteActionSetFromHDFSFunctionsTest.java

@@ -2,9 +2,8 @@ package eu.dnetlib.dhp.actionmanager;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
+import eu.dnetlib.dhp.schema.oaf.Oaf;
 import org.apache.spark.SparkConf;
-import org.apache.spark.sql.Column;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SparkSession;
@@ -13,13 +12,11 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
 import java.util.function.BiFunction;
-import java.util.stream.Collectors;
+import java.util.function.Function;
 
-import static org.apache.spark.sql.functions.get_json_object;
 import static org.junit.Assert.assertEquals;
 
 public class PromoteActionSetFromHDFSFunctionsTest {
@@ -32,6 +29,7 @@ public class PromoteActionSetFromHDFSFunctionsTest {
         conf.setMaster("local");
         conf.setAppName(PromoteActionSetFromHDFSFunctionsTest.class.getSimpleName());
         conf.set("spark.driver.host", "localhost");
+
         spark = SparkSession.builder().config(conf).getOrCreate();
     }
 
@@ -40,24 +38,93 @@ public class PromoteActionSetFromHDFSFunctionsTest {
         spark.stop();
     }
 
+    @Test
+    public void shouldJoinWithActionPayloadUsingIdAndMerge() {
+        // given
+        String id1 = "id1";
+        String id2 = "id2";
+        String id3 = "id3";
+        String id4 = "id4";
+        List<OafImpl> oafData = Arrays.asList(
+                createOafImpl(id1), createOafImpl(id2), createOafImpl(id3), createOafImpl(id4)
+        );
+        Dataset<OafImpl> oafDS = spark.createDataset(oafData, Encoders.bean(OafImpl.class));
+
+        List<String> actionPayloadData = Arrays.asList(
+                createActionPayload(id1),
+                createActionPayload(id2), createActionPayload(id2),
+                createActionPayload(id3), createActionPayload(id3), createActionPayload(id3)
+        );
+        Dataset<String> actionPayloadDS = spark.createDataset(actionPayloadData, Encoders.STRING());
+
+        SerializableSupplier<Function<OafImpl, String>> oafIdFn = () -> OafImpl::getId;
+        SerializableSupplier<BiFunction<String, Class<OafImpl>, OafImpl>> actionPayloadToOafFn = () -> (s, clazz) -> {
+            try {
+                JsonNode jsonNode = new ObjectMapper().readTree(s);
+                String id = jsonNode.at("/id").asText();
+                return createOafImpl(id);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        };
+        SerializableSupplier<BiFunction<OafImpl, OafImpl, OafImpl>> mergeAndGetFn = () -> (x, y) -> {
+            x.mergeFrom(y);
+            return x;
+        };
+
+        // when
+        List<OafImpl> results = PromoteActionSetFromHDFSFunctions
+                .joinOafEntityWithActionPayloadAndMerge(oafDS,
+                        actionPayloadDS,
+                        oafIdFn,
+                        actionPayloadToOafFn,
+                        mergeAndGetFn,
+                        OafImpl.class)
+                .collectAsList();
+//        System.out.println(results.stream().map(x -> String.format("%s:%d", x.getId(), x.merged)).collect(Collectors.joining(",")));
+
+        // then
+        assertEquals(7, results.size());
+        results.forEach(result -> {
+            switch (result.getId()) {
+                case "id1":
+                case "id2":
+                case "id3":
+                    assertEquals(2, result.merged);
+                    break;
+                case "id4":
+                    assertEquals(1, result.merged);
+                    break;
+            }
+        });
+    }
+
     @Test
     public void shouldGroupOafEntitiesByIdAndMergeWithinGroup() {
         // given
         String id1 = "id1";
         String id2 = "id2";
         String id3 = "id3";
-        List<OafEntityImpl> entityData = Arrays.asList(
+        List<OafImpl> oafData = Arrays.asList(
-                createOafEntityImpl(id1),
+                createOafImpl(id1),
-                createOafEntityImpl(id2), createOafEntityImpl(id2),
+                createOafImpl(id2), createOafImpl(id2),
-                createOafEntityImpl(id3), createOafEntityImpl(id3), createOafEntityImpl(id3)
+                createOafImpl(id3), createOafImpl(id3), createOafImpl(id3)
         );
-        Dataset<OafEntityImpl> entityDS = spark.createDataset(entityData, Encoders.bean(OafEntityImpl.class));
+        Dataset<OafImpl> oafDS = spark.createDataset(oafData, Encoders.bean(OafImpl.class));
+        SerializableSupplier<Function<OafImpl, String>> idFn = () -> OafImpl::getId;
+        SerializableSupplier<BiFunction<OafImpl, OafImpl, OafImpl>> mergeAndGetFn = () -> (x, y) -> {
+            x.mergeFrom(y);
+            return x;
+        };
 
         // when
-        List<OafEntityImpl> results = PromoteActionSetFromHDFSFunctions
+        List<OafImpl> results = PromoteActionSetFromHDFSFunctions
-                .groupEntitiesByIdAndMerge(entityDS, OafEntityImpl.class)
+                .groupOafByIdAndMerge(oafDS,
+                        idFn,
+                        mergeAndGetFn,
+                        OafImpl.class)
                 .collectAsList();
-        System.out.println(results.stream().map(x -> String.format("%s:%d", x.getId(), x.merged)).collect(Collectors.joining(",")));
+//        System.out.println(results.stream().map(x -> String.format("%s:%d", x.getId(), x.merged)).collect(Collectors.joining(",")));
 
         // then
         assertEquals(3, results.size());
@@ -76,76 +143,20 @@ public class PromoteActionSetFromHDFSFunctionsTest {
         });
     }
 
-    @Test
+    public static class OafImpl extends Oaf {
-    public void shouldJoinWithActionPayloadUsingIdAndMerge() {
+        private String id;
-        // given
-        String id1 = "id1";
-        String id2 = "id2";
-        String id3 = "id3";
-        String id4 = "id4";
-        List<OafEntityImpl> entityData = Arrays.asList(
-                createOafEntityImpl(id1), createOafEntityImpl(id2), createOafEntityImpl(id3), createOafEntityImpl(id4)
-        );
-        Dataset<OafEntityImpl> entityDS = spark.createDataset(entityData, Encoders.bean(OafEntityImpl.class));
-
-        List<String> actionPayloadData = Arrays.asList(
-                actionPayload(id1),
-                actionPayload(id2), actionPayload(id2),
-                actionPayload(id3), actionPayload(id3), actionPayload(id3)
-        );
-        Dataset<String> actionPayloadDS = spark.createDataset(actionPayloadData, Encoders.STRING());
-
-        BiFunction<Dataset<OafEntityImpl>, Dataset<String>, Column> entityToActionPayloadJoinExpr = (left, right) ->
-                left.col("id").equalTo(get_json_object(right.col("value"), "$.id"));
-        BiFunction<String, Class<OafEntityImpl>, OafEntityImpl> actionPayloadToEntityFn =
-                (BiFunction<String, Class<OafEntityImpl>, OafEntityImpl> & Serializable) (s, clazz) -> {
-                    try {
-                        JsonNode jsonNode = new ObjectMapper().readTree(s);
-                        String id = jsonNode.at("/id").asText();
-                        OafEntityImpl x = new OafEntityImpl();
-                        x.setId(id);
-                        return x;
-                    } catch (IOException e) {
-                        throw new RuntimeException(e);
-                    }
-                };
-
-        // when
-        List<OafEntityImpl> results = PromoteActionSetFromHDFSFunctions
-                .joinEntitiesWithActionPayloadAndMerge(entityDS,
-                        actionPayloadDS,
-                        entityToActionPayloadJoinExpr,
-                        actionPayloadToEntityFn,
-                        OafEntityImpl.class)
-                .collectAsList();
-        System.out.println(results.stream().map(x -> String.format("%s:%d", x.getId(), x.merged)).collect(Collectors.joining(",")));
-
-        // then
-        assertEquals(7, results.size());
-        results.forEach(result -> {
-            switch (result.getId()) {
-                case "id1":
-                    assertEquals(2, result.merged);
-                    break;
-                case "id2":
-                    assertEquals(2, result.merged);
-                    break;
-                case "id3":
-                    assertEquals(2, result.merged);
-                    break;
-                case "id4":
-                    assertEquals(1, result.merged);
-                    break;
-            }
-        });
-    }
-
-    public static class OafEntityImpl extends OafEntity {
         private int merged = 1;
 
-        @Override
+        public void mergeFrom(Oaf e) {
-        public void mergeFrom(OafEntity e) {
+            merged += ((OafImpl) e).merged;
-            merged += ((OafEntityImpl) e).merged;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public void setId(String id) {
+            this.id = id;
         }
 
         public int getMerged() {
@@ -157,13 +168,13 @@ public class PromoteActionSetFromHDFSFunctionsTest {
         }
     }
 
-    private static OafEntityImpl createOafEntityImpl(String id) {
+    private static OafImpl createOafImpl(String id) {
-        OafEntityImpl x = new OafEntityImpl();
+        OafImpl x = new OafImpl();
         x.setId(id);
         return x;
     }
 
-    private static String actionPayload(String id) {
+    private static String createActionPayload(String id) {
         return String.format("{\"id\":\"%s\"}", id);
     }
 }

dhp-workflows/dhp-actionmanager/src/test/java/eu/dnetlib/dhp/actionmanager/PromoteActionSetFromHDFSJobTest.java

@@ -1,79 +1,269 @@
 package eu.dnetlib.dhp.actionmanager;
 
-import org.junit.Before;
+import com.fasterxml.jackson.databind.DeserializationFeature;
-import org.junit.Test;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import eu.dnetlib.dhp.schema.oaf.*;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.junit.*;
+import scala.Tuple2;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.spark.sql.functions.base64;
+import static org.apache.spark.sql.functions.col;
+import static org.junit.Assert.*;
 
 public class PromoteActionSetFromHDFSJobTest {
-    private ClassLoader cl = getClass().getClassLoader();
+    private static SparkSession spark;
+    private static Configuration configuration;
+
     private Path workingDir;
     private Path inputDir;
+    private Path inputGraphDir;
+    private Path inputActionSetsDir;
     private Path outputDir;
 
+    private ClassLoader cl = getClass().getClassLoader();
+
+    @BeforeClass
+    public static void beforeClass() throws IOException {
+        SparkConf conf = new SparkConf();
+        conf.setAppName(PromoteActionSetFromHDFSJobTest.class.getSimpleName());
+        conf.setMaster("local");
+        conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+        conf.registerKryoClasses(new Class[]{
+                Author.class,
+                Context.class,
+                Country.class,
+                DataInfo.class,
+                eu.dnetlib.dhp.schema.oaf.Dataset.class,
+                Datasource.class,
+                ExternalReference.class,
+                ExtraInfo.class,
+                Field.class,
+                GeoLocation.class,
+                Instance.class,
+                Journal.class,
+                KeyValue.class,
+                Oaf.class,
+                OafEntity.class,
+                OAIProvenance.class,
+                Organization.class,
+                OriginDescription.class,
+                OtherResearchProduct.class,
+                Project.class,
+                Publication.class,
+                Qualifier.class,
+                Relation.class,
+                Result.class,
+                Software.class,
+                StructuredProperty.class
+        });
+
+        spark = SparkSession.builder().config(conf).getOrCreate();
+
+        configuration = Job.getInstance().getConfiguration();
+    }
+
     @Before
     public void before() throws IOException {
         workingDir = Files.createTempDirectory("promote_action_set");
         inputDir = workingDir.resolve("input");
+        inputGraphDir = inputDir.resolve("graph");
+
+        inputActionSetsDir = inputDir.resolve("action_sets");
+
         outputDir = workingDir.resolve("output");
     }
 
-//    @After
+    @After
-//    public void after() throws IOException {
+    public void after() throws IOException {
-//        FileUtils.deleteDirectory(workingDir.toFile());
+        FileUtils.deleteDirectory(workingDir.toFile());
-//    }
+    }
+
+    @AfterClass
+    public static void afterAll() {
+        spark.stop();
+    }
 
     @Test
-    public void shouldReadAtomicActionsFromHDFSAndWritePartitionedAsParquetFiles() throws Exception {
+    public void shouldThrowWhenGraphTableIsInvalid() {
+        fail("not yet implemented");
+    }
+
+    @Test
+    public void shouldReadActionsFromHDFSAndPromoteThemForDatasetsUsingMergeFromStrategy() throws Exception {
+        readActionsFromHDFSAndPromoteThemFor("dataset",
+                OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
+                eu.dnetlib.dhp.schema.oaf.Dataset.class);
+    }
+
+    @Test
+    public void shouldReadActionsFromHDFSAndPromoteThemForDatasourcesUsingMergeFromStrategy() throws Exception {
+        readActionsFromHDFSAndPromoteThemFor("datasource",
+                OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
+                Datasource.class);
+    }
+
+    @Test
+    public void shouldReadActionsFromHDFSAndPromoteThemForOrganizationUsingMergeFromStrategy() throws Exception {
+        readActionsFromHDFSAndPromoteThemFor("organization",
+                OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
+                Organization.class);
+    }
+
+    @Test
+    public void shouldReadActionsFromHDFSAndPromoteThemForOtherResearchProductUsingMergeFromStrategy() throws Exception {
+        readActionsFromHDFSAndPromoteThemFor("otherresearchproduct",
+                OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
+                OtherResearchProduct.class);
+    }
+
+    @Test
+    public void shouldReadActionsFromHDFSAndPromoteThemForProjectUsingMergeFromStrategy() throws Exception {
+        readActionsFromHDFSAndPromoteThemFor("project",
+                OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
+                Project.class);
+    }
+
+    @Test
+    public void shouldReadActionsFromHDFSAndPromoteThemForPublicationUsingMergeFromStrategy() throws Exception {
+        readActionsFromHDFSAndPromoteThemFor("publication",
+                OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
+                Publication.class);
+    }
+
+    @Test
+    public void shouldReadActionsFromHDFSAndPromoteThemForRelationUsingMergeFromStrategy() throws Exception {
+        readActionsFromHDFSAndPromoteThemFor("relation",
+                OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
+                Relation.class);
+    }
+
+    @Test
+    public void shouldReadActionsFromHDFSAndPromoteThemForSoftwareUsingMergeFromStrategy() throws Exception {
+        readActionsFromHDFSAndPromoteThemFor("software",
+                OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
+                Software.class);
+    }
+
+    private <T extends Oaf> void readActionsFromHDFSAndPromoteThemFor(String graphTableName,
+                                                                      OafMergeAndGet.Strategy strategy,
+                                                                      Class<T> clazz) throws Exception {
         // given
-        // NOTE: test resource should contain atomic actions in a human readable form, probably as json files; here the
+        String inputGraphTableJsonDumpPath =
-        // files should be converted to a serialized format and written out to workingDir/input
+                String.format("%s/%s.json", "eu/dnetlib/dhp/actionmanager/input/graph", graphTableName);
-        // for current testing: actions from iis export, given as sequence file are copied to workingDir/input/
+        createGraphTableFor(inputGraphTableJsonDumpPath, graphTableName, clazz);
-
+        String inputActionSetPaths = createActionSets();
-        //graph
+        Path outputGraphDir = outputDir.resolve("graph");
-        Path inputGraphDir = inputDir.resolve("graph");
-        Files.createDirectories(inputGraphDir);
-        copyFiles(Paths.get(Objects.requireNonNull(cl.getResource("graph")).getFile()), inputGraphDir);
-
-        //actions
-        Path inputActionsDir = inputDir.resolve("actions");
-        Files.createDirectories(inputActionsDir);
-
-        Path inputEntitiesPatentDir = inputActionsDir.resolve("entities_patent");
-        Files.createDirectories(inputEntitiesPatentDir);
-        copyFiles(Paths.get(Objects.requireNonNull(cl.getResource("actions/entities_patent")).getFile()), inputEntitiesPatentDir);
-
-        Path inputEntitiesSoftwareDir = inputActionsDir.resolve("entities_software");
-        Files.createDirectories(inputEntitiesSoftwareDir);
-        copyFiles(Paths.get(Objects.requireNonNull(cl.getResource("actions/entities_software")).getFile()), inputEntitiesSoftwareDir);
-
-        String inputActionSetPaths = String.join(",",   inputEntitiesSoftwareDir.toString()); //inputEntitiesPatentDir.toString(),
 
+        // when
         PromoteActionSetFromHDFSJob.main(new String[]{
-                "-master", "local[*]",
+                "-isSparkSessionManaged", Boolean.FALSE.toString(),
                 "-inputGraphPath", inputGraphDir.toString(),
                 "-inputActionSetPaths", inputActionSetPaths,
-                "-outputGraphPath", outputDir.toString()
+                "-graphTableName", graphTableName,
+                "-outputGraphPath", outputGraphDir.toString(),
+                "mergeAndGetStrategy", strategy.name()
         });
+
+        // then
+        Path outputGraphTableDir = outputGraphDir.resolve(graphTableName);
+        assertTrue(Files.exists(outputGraphDir));
+
+        List<T> outputGraphTableRows = readGraphTableFromParquet(outputGraphTableDir.toString(), clazz).collectAsList();
+        outputGraphTableRows.sort(Comparator.comparingInt(Object::hashCode));
+
+        assertEquals(10, outputGraphTableRows.size());
+
+        String expectedOutputGraphTableJsonDumpPath =
+                String.format("%s/%s/%s.json", "eu/dnetlib/dhp/actionmanager/output/graph", strategy.name().toLowerCase(), graphTableName);
+        Path expectedOutputGraphTableJsonDumpFile = Paths
+                .get(Objects.requireNonNull(cl.getResource(expectedOutputGraphTableJsonDumpPath)).getFile());
+        List<T> expectedOutputGraphTableRows = readGraphTableFromJSON(
+                expectedOutputGraphTableJsonDumpFile, clazz)
+                .collectAsList();
+        expectedOutputGraphTableRows.sort(Comparator.comparingInt(Object::hashCode));
+
+        assertArrayEquals(expectedOutputGraphTableRows.toArray(), outputGraphTableRows.toArray());
     }
 
-    private static void copyFiles(Path source, Path target) throws IOException {
+    private <T extends Oaf> void createGraphTableFor(String inputGraphTableJsonDumpPath,
-        Files.list(source).forEach(f -> {
+                                                     String inputGraphTableDirRelativePath,
-            try {
+                                                     Class<T> clazz) {
-                if (Files.isDirectory(f)) {
+        Path inputGraphTableJsonDumpFile = Paths
-                    Path subTarget = Files.createDirectories(target.resolve(f.getFileName()));
+                .get(Objects.requireNonNull(cl.getResource(inputGraphTableJsonDumpPath)).getFile());
-                    copyFiles(f, subTarget);
+        Dataset<T> inputGraphTableDS = readGraphTableFromJSON(inputGraphTableJsonDumpFile, clazz);
-                } else {
+        Path inputGraphTableDir = inputGraphDir.resolve(inputGraphTableDirRelativePath);
-                    Files.copy(f, target.resolve(f.getFileName()));
+        inputGraphTableDS
-                }
+                .toJSON()
-            } catch (IOException e) {
+                .javaRDD()
-                e.printStackTrace();
+                .mapToPair(json -> new Tuple2<>(new Text(clazz.getCanonicalName()), new Text(json)))
-                throw new RuntimeException(e);
+                .saveAsNewAPIHadoopFile(inputGraphTableDir.toString(),
+                        Text.class,
+                        Text.class,
+                        SequenceFileOutputFormat.class,
+                        configuration);
     }
+
+    private String createActionSets() throws IOException {
+        Path inputActionPayloadJsonDumpsDir = Paths
+                .get(Objects.requireNonNull(cl.getResource("eu/dnetlib/dhp/actionmanager/input/action_sets/"))
+                        .getFile());
+        Files
+                .list(inputActionPayloadJsonDumpsDir)
+                .forEach(inputActionPayloadJsonDump -> {
+                    String inputActionSetId = inputActionPayloadJsonDump.getFileName().toString();
+                    Path inputActionSetDir = inputActionSetsDir.resolve(inputActionPayloadJsonDump.getFileName());
+                    spark.read()
+                            .textFile(inputActionPayloadJsonDump.toString())
+                            .withColumn("TargetValue", base64(col("value")))
+                            .select("TargetValue")
+                            .toJSON()
+                            .javaRDD()
+                            .mapToPair(json -> new Tuple2<>(new Text(inputActionSetId), new Text(json)))
+                            .saveAsNewAPIHadoopFile(inputActionSetDir.toString(),
+                                    Text.class,
+                                    Text.class,
+                                    SequenceFileOutputFormat.class,
+                                    configuration);
                 });
+        return Files.list(inputActionSetsDir).map(Path::toString).collect(Collectors.joining(","));
+    }
+
+    private static <T extends Oaf> Dataset<T> readGraphTableFromJSON(Path path, Class<T> clazz) {
+        return spark.read()
+                .format("json")
+                .load(path.toString())
+                .toJSON()
+                .map((MapFunction<String, T>) json -> newObjectMapper().readValue(json, clazz), Encoders.bean(clazz));
+    }
+
+    //TODO: enable strict deserialization later
+    private static ObjectMapper newObjectMapper() {
+        return new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+    }
+
+    private static <T extends Oaf> Dataset<T> readGraphTableFromParquet(String outputGraphTablePath, Class<T> clazz) {
+        return spark.read()
+                .format("parquet")
+                .load(outputGraphTablePath)
+                .as(Encoders.bean(clazz));
     }
 }

dhp-workflows/dhp-actionmanager/src/test/resources/eu/dnetlib/dhp/actionmanager/input/action_sets/payload_relation.json

@@ -0,0 +1,20 @@
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::007a4870b31056f89b768cf508e1538e"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::007a4870b31056f89b768cf508e1538e"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::9ea9c0996c87e1dc7fc69f94b5ed0010"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::9ea9c0996c87e1dc7fc69f94b5ed0010"}
+{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::d0bbea1f5bed5864d1904eb602e608a6"}
+{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::d0bbea1f5bed5864d1904eb602e608a6"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","subRelType":"provision","target":"20|openaire____::c24a458004a31f9687089ea3d249de51"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","subRelType":"provision","target":"20|openaire____::c24a458004a31f9687089ea3d249de51"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::8de0f5a712997aafe0d794a53e51b75a"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::8de0f5a712997aafe0d794a53e51b75a"}
+{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::b58bdbe8ae5acead04fc76777d2f8017"}
+{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::b58bdbe8ae5acead04fc76777d2f8017"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|NARCIS__cris::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::c978e29d3b2ddf4f0c2b6e60d6613426"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|NARCIS__cris::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::c978e29d3b2ddf4f0c2b6e60d6613426"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|OpenstarTs__::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::fc7459b8fed8c0d47947fe04275251c0"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|OpenstarTs__::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::fc7459b8fed8c0d47947fe04275251c0"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|UnityFVG____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::89bab7c5a227fc27b2b9cadf475a6b71"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|UnityFVG____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::89bab7c5a227fc27b2b9cadf475a6b71"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|VTTRsInSsCrs::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::735915884eb439d42953372eaf934782"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|VTTRsInSsCrs::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::735915884eb439d42953372eaf934782"}

dhp-workflows/dhp-actionmanager/src/test/resources/eu/dnetlib/dhp/actionmanager/input/graph/relation.json

@@ -0,0 +1,10 @@
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::007a4870b31056f89b768cf508e1538e"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::9ea9c0996c87e1dc7fc69f94b5ed0010"}
+{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::d0bbea1f5bed5864d1904eb602e608a6"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","subRelType":"provision","target":"20|openaire____::c24a458004a31f9687089ea3d249de51"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::8de0f5a712997aafe0d794a53e51b75a"}
+{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::b58bdbe8ae5acead04fc76777d2f8017"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|NARCIS__cris::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::c978e29d3b2ddf4f0c2b6e60d6613426"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|OpenstarTs__::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::fc7459b8fed8c0d47947fe04275251c0"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|UnityFVG____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::89bab7c5a227fc27b2b9cadf475a6b71"}
+{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|VTTRsInSsCrs::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::735915884eb439d42953372eaf934782"}

dhp-workflows/dhp-actionmanager/src/test/resources/eu/dnetlib/dhp/actionmanager/output/graph/merge_from_and_get/relation.json

@@ -0,0 +1,10 @@
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::d0bbea1f5bed5864d1904eb602e608a6"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|OpenstarTs__::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::fc7459b8fed8c0d47947fe04275251c0"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|NARCIS__cris::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::c978e29d3b2ddf4f0c2b6e60d6613426"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::b58bdbe8ae5acead04fc76777d2f8017"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::8de0f5a712997aafe0d794a53e51b75a"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|UnityFVG____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::89bab7c5a227fc27b2b9cadf475a6b71"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::007a4870b31056f89b768cf508e1538e"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|VTTRsInSsCrs::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::735915884eb439d42953372eaf934782"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::9ea9c0996c87e1dc7fc69f94b5ed0010"}
+{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","subRelType":"provision","target":"20|openaire____::c24a458004a31f9687089ea3d249de51"}