forked from D-Net/dnet-hadoop
WIP: promote job tests and test resources implementation snapshot
This commit is contained in:
parent
2e996d610f
commit
3f24593e51
|
|
@ -2,9 +2,8 @@ package eu.dnetlib.dhp.actionmanager;
|
|||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
||||
import eu.dnetlib.dhp.schema.oaf.Oaf;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.sql.Column;
|
||||
import org.apache.spark.sql.Dataset;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
|
|
@ -13,13 +12,11 @@ import org.junit.BeforeClass;
|
|||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static org.apache.spark.sql.functions.get_json_object;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class PromoteActionSetFromHDFSFunctionsTest {
|
||||
|
|
@ -32,6 +29,7 @@ public class PromoteActionSetFromHDFSFunctionsTest {
|
|||
conf.setMaster("local");
|
||||
conf.setAppName(PromoteActionSetFromHDFSFunctionsTest.class.getSimpleName());
|
||||
conf.set("spark.driver.host", "localhost");
|
||||
|
||||
spark = SparkSession.builder().config(conf).getOrCreate();
|
||||
}
|
||||
|
||||
|
|
@ -40,24 +38,93 @@ public class PromoteActionSetFromHDFSFunctionsTest {
|
|||
spark.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldJoinWithActionPayloadUsingIdAndMerge() {
|
||||
// given
|
||||
String id1 = "id1";
|
||||
String id2 = "id2";
|
||||
String id3 = "id3";
|
||||
String id4 = "id4";
|
||||
List<OafImpl> oafData = Arrays.asList(
|
||||
createOafImpl(id1), createOafImpl(id2), createOafImpl(id3), createOafImpl(id4)
|
||||
);
|
||||
Dataset<OafImpl> oafDS = spark.createDataset(oafData, Encoders.bean(OafImpl.class));
|
||||
|
||||
List<String> actionPayloadData = Arrays.asList(
|
||||
createActionPayload(id1),
|
||||
createActionPayload(id2), createActionPayload(id2),
|
||||
createActionPayload(id3), createActionPayload(id3), createActionPayload(id3)
|
||||
);
|
||||
Dataset<String> actionPayloadDS = spark.createDataset(actionPayloadData, Encoders.STRING());
|
||||
|
||||
SerializableSupplier<Function<OafImpl, String>> oafIdFn = () -> OafImpl::getId;
|
||||
SerializableSupplier<BiFunction<String, Class<OafImpl>, OafImpl>> actionPayloadToOafFn = () -> (s, clazz) -> {
|
||||
try {
|
||||
JsonNode jsonNode = new ObjectMapper().readTree(s);
|
||||
String id = jsonNode.at("/id").asText();
|
||||
return createOafImpl(id);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
};
|
||||
SerializableSupplier<BiFunction<OafImpl, OafImpl, OafImpl>> mergeAndGetFn = () -> (x, y) -> {
|
||||
x.mergeFrom(y);
|
||||
return x;
|
||||
};
|
||||
|
||||
// when
|
||||
List<OafImpl> results = PromoteActionSetFromHDFSFunctions
|
||||
.joinOafEntityWithActionPayloadAndMerge(oafDS,
|
||||
actionPayloadDS,
|
||||
oafIdFn,
|
||||
actionPayloadToOafFn,
|
||||
mergeAndGetFn,
|
||||
OafImpl.class)
|
||||
.collectAsList();
|
||||
// System.out.println(results.stream().map(x -> String.format("%s:%d", x.getId(), x.merged)).collect(Collectors.joining(",")));
|
||||
|
||||
// then
|
||||
assertEquals(7, results.size());
|
||||
results.forEach(result -> {
|
||||
switch (result.getId()) {
|
||||
case "id1":
|
||||
case "id2":
|
||||
case "id3":
|
||||
assertEquals(2, result.merged);
|
||||
break;
|
||||
case "id4":
|
||||
assertEquals(1, result.merged);
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldGroupOafEntitiesByIdAndMergeWithinGroup() {
|
||||
// given
|
||||
String id1 = "id1";
|
||||
String id2 = "id2";
|
||||
String id3 = "id3";
|
||||
List<OafEntityImpl> entityData = Arrays.asList(
|
||||
createOafEntityImpl(id1),
|
||||
createOafEntityImpl(id2), createOafEntityImpl(id2),
|
||||
createOafEntityImpl(id3), createOafEntityImpl(id3), createOafEntityImpl(id3)
|
||||
List<OafImpl> oafData = Arrays.asList(
|
||||
createOafImpl(id1),
|
||||
createOafImpl(id2), createOafImpl(id2),
|
||||
createOafImpl(id3), createOafImpl(id3), createOafImpl(id3)
|
||||
);
|
||||
Dataset<OafEntityImpl> entityDS = spark.createDataset(entityData, Encoders.bean(OafEntityImpl.class));
|
||||
Dataset<OafImpl> oafDS = spark.createDataset(oafData, Encoders.bean(OafImpl.class));
|
||||
SerializableSupplier<Function<OafImpl, String>> idFn = () -> OafImpl::getId;
|
||||
SerializableSupplier<BiFunction<OafImpl, OafImpl, OafImpl>> mergeAndGetFn = () -> (x, y) -> {
|
||||
x.mergeFrom(y);
|
||||
return x;
|
||||
};
|
||||
|
||||
// when
|
||||
List<OafEntityImpl> results = PromoteActionSetFromHDFSFunctions
|
||||
.groupEntitiesByIdAndMerge(entityDS, OafEntityImpl.class)
|
||||
List<OafImpl> results = PromoteActionSetFromHDFSFunctions
|
||||
.groupOafByIdAndMerge(oafDS,
|
||||
idFn,
|
||||
mergeAndGetFn,
|
||||
OafImpl.class)
|
||||
.collectAsList();
|
||||
System.out.println(results.stream().map(x -> String.format("%s:%d", x.getId(), x.merged)).collect(Collectors.joining(",")));
|
||||
// System.out.println(results.stream().map(x -> String.format("%s:%d", x.getId(), x.merged)).collect(Collectors.joining(",")));
|
||||
|
||||
// then
|
||||
assertEquals(3, results.size());
|
||||
|
|
@ -76,76 +143,20 @@ public class PromoteActionSetFromHDFSFunctionsTest {
|
|||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldJoinWithActionPayloadUsingIdAndMerge() {
|
||||
// given
|
||||
String id1 = "id1";
|
||||
String id2 = "id2";
|
||||
String id3 = "id3";
|
||||
String id4 = "id4";
|
||||
List<OafEntityImpl> entityData = Arrays.asList(
|
||||
createOafEntityImpl(id1), createOafEntityImpl(id2), createOafEntityImpl(id3), createOafEntityImpl(id4)
|
||||
);
|
||||
Dataset<OafEntityImpl> entityDS = spark.createDataset(entityData, Encoders.bean(OafEntityImpl.class));
|
||||
|
||||
List<String> actionPayloadData = Arrays.asList(
|
||||
actionPayload(id1),
|
||||
actionPayload(id2), actionPayload(id2),
|
||||
actionPayload(id3), actionPayload(id3), actionPayload(id3)
|
||||
);
|
||||
Dataset<String> actionPayloadDS = spark.createDataset(actionPayloadData, Encoders.STRING());
|
||||
|
||||
BiFunction<Dataset<OafEntityImpl>, Dataset<String>, Column> entityToActionPayloadJoinExpr = (left, right) ->
|
||||
left.col("id").equalTo(get_json_object(right.col("value"), "$.id"));
|
||||
BiFunction<String, Class<OafEntityImpl>, OafEntityImpl> actionPayloadToEntityFn =
|
||||
(BiFunction<String, Class<OafEntityImpl>, OafEntityImpl> & Serializable) (s, clazz) -> {
|
||||
try {
|
||||
JsonNode jsonNode = new ObjectMapper().readTree(s);
|
||||
String id = jsonNode.at("/id").asText();
|
||||
OafEntityImpl x = new OafEntityImpl();
|
||||
x.setId(id);
|
||||
return x;
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
};
|
||||
|
||||
// when
|
||||
List<OafEntityImpl> results = PromoteActionSetFromHDFSFunctions
|
||||
.joinEntitiesWithActionPayloadAndMerge(entityDS,
|
||||
actionPayloadDS,
|
||||
entityToActionPayloadJoinExpr,
|
||||
actionPayloadToEntityFn,
|
||||
OafEntityImpl.class)
|
||||
.collectAsList();
|
||||
System.out.println(results.stream().map(x -> String.format("%s:%d", x.getId(), x.merged)).collect(Collectors.joining(",")));
|
||||
|
||||
// then
|
||||
assertEquals(7, results.size());
|
||||
results.forEach(result -> {
|
||||
switch (result.getId()) {
|
||||
case "id1":
|
||||
assertEquals(2, result.merged);
|
||||
break;
|
||||
case "id2":
|
||||
assertEquals(2, result.merged);
|
||||
break;
|
||||
case "id3":
|
||||
assertEquals(2, result.merged);
|
||||
break;
|
||||
case "id4":
|
||||
assertEquals(1, result.merged);
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public static class OafEntityImpl extends OafEntity {
|
||||
public static class OafImpl extends Oaf {
|
||||
private String id;
|
||||
private int merged = 1;
|
||||
|
||||
@Override
|
||||
public void mergeFrom(OafEntity e) {
|
||||
merged += ((OafEntityImpl) e).merged;
|
||||
public void mergeFrom(Oaf e) {
|
||||
merged += ((OafImpl) e).merged;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public int getMerged() {
|
||||
|
|
@ -157,13 +168,13 @@ public class PromoteActionSetFromHDFSFunctionsTest {
|
|||
}
|
||||
}
|
||||
|
||||
private static OafEntityImpl createOafEntityImpl(String id) {
|
||||
OafEntityImpl x = new OafEntityImpl();
|
||||
private static OafImpl createOafImpl(String id) {
|
||||
OafImpl x = new OafImpl();
|
||||
x.setId(id);
|
||||
return x;
|
||||
}
|
||||
|
||||
private static String actionPayload(String id) {
|
||||
private static String createActionPayload(String id) {
|
||||
return String.format("{\"id\":\"%s\"}", id);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,79 +1,269 @@
|
|||
package eu.dnetlib.dhp.actionmanager;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import eu.dnetlib.dhp.schema.oaf.*;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.mapreduce.Job;
|
||||
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.function.MapFunction;
|
||||
import org.apache.spark.sql.Dataset;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
import org.junit.*;
|
||||
import scala.Tuple2;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.apache.spark.sql.functions.base64;
|
||||
import static org.apache.spark.sql.functions.col;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class PromoteActionSetFromHDFSJobTest {
|
||||
private ClassLoader cl = getClass().getClassLoader();
|
||||
private static SparkSession spark;
|
||||
private static Configuration configuration;
|
||||
|
||||
private Path workingDir;
|
||||
private Path inputDir;
|
||||
private Path inputGraphDir;
|
||||
private Path inputActionSetsDir;
|
||||
private Path outputDir;
|
||||
|
||||
private ClassLoader cl = getClass().getClassLoader();
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws IOException {
|
||||
SparkConf conf = new SparkConf();
|
||||
conf.setAppName(PromoteActionSetFromHDFSJobTest.class.getSimpleName());
|
||||
conf.setMaster("local");
|
||||
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
|
||||
conf.registerKryoClasses(new Class[]{
|
||||
Author.class,
|
||||
Context.class,
|
||||
Country.class,
|
||||
DataInfo.class,
|
||||
eu.dnetlib.dhp.schema.oaf.Dataset.class,
|
||||
Datasource.class,
|
||||
ExternalReference.class,
|
||||
ExtraInfo.class,
|
||||
Field.class,
|
||||
GeoLocation.class,
|
||||
Instance.class,
|
||||
Journal.class,
|
||||
KeyValue.class,
|
||||
Oaf.class,
|
||||
OafEntity.class,
|
||||
OAIProvenance.class,
|
||||
Organization.class,
|
||||
OriginDescription.class,
|
||||
OtherResearchProduct.class,
|
||||
Project.class,
|
||||
Publication.class,
|
||||
Qualifier.class,
|
||||
Relation.class,
|
||||
Result.class,
|
||||
Software.class,
|
||||
StructuredProperty.class
|
||||
});
|
||||
|
||||
spark = SparkSession.builder().config(conf).getOrCreate();
|
||||
|
||||
configuration = Job.getInstance().getConfiguration();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void before() throws IOException {
|
||||
workingDir = Files.createTempDirectory("promote_action_set");
|
||||
inputDir = workingDir.resolve("input");
|
||||
inputGraphDir = inputDir.resolve("graph");
|
||||
|
||||
inputActionSetsDir = inputDir.resolve("action_sets");
|
||||
|
||||
outputDir = workingDir.resolve("output");
|
||||
}
|
||||
|
||||
// @After
|
||||
// public void after() throws IOException {
|
||||
// FileUtils.deleteDirectory(workingDir.toFile());
|
||||
// }
|
||||
@After
|
||||
public void after() throws IOException {
|
||||
FileUtils.deleteDirectory(workingDir.toFile());
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterAll() {
|
||||
spark.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadAtomicActionsFromHDFSAndWritePartitionedAsParquetFiles() throws Exception {
|
||||
public void shouldThrowWhenGraphTableIsInvalid() {
|
||||
fail("not yet implemented");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadActionsFromHDFSAndPromoteThemForDatasetsUsingMergeFromStrategy() throws Exception {
|
||||
readActionsFromHDFSAndPromoteThemFor("dataset",
|
||||
OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
eu.dnetlib.dhp.schema.oaf.Dataset.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadActionsFromHDFSAndPromoteThemForDatasourcesUsingMergeFromStrategy() throws Exception {
|
||||
readActionsFromHDFSAndPromoteThemFor("datasource",
|
||||
OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
Datasource.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadActionsFromHDFSAndPromoteThemForOrganizationUsingMergeFromStrategy() throws Exception {
|
||||
readActionsFromHDFSAndPromoteThemFor("organization",
|
||||
OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
Organization.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadActionsFromHDFSAndPromoteThemForOtherResearchProductUsingMergeFromStrategy() throws Exception {
|
||||
readActionsFromHDFSAndPromoteThemFor("otherresearchproduct",
|
||||
OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
OtherResearchProduct.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadActionsFromHDFSAndPromoteThemForProjectUsingMergeFromStrategy() throws Exception {
|
||||
readActionsFromHDFSAndPromoteThemFor("project",
|
||||
OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
Project.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadActionsFromHDFSAndPromoteThemForPublicationUsingMergeFromStrategy() throws Exception {
|
||||
readActionsFromHDFSAndPromoteThemFor("publication",
|
||||
OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
Publication.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadActionsFromHDFSAndPromoteThemForRelationUsingMergeFromStrategy() throws Exception {
|
||||
readActionsFromHDFSAndPromoteThemFor("relation",
|
||||
OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
Relation.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReadActionsFromHDFSAndPromoteThemForSoftwareUsingMergeFromStrategy() throws Exception {
|
||||
readActionsFromHDFSAndPromoteThemFor("software",
|
||||
OafMergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
Software.class);
|
||||
}
|
||||
|
||||
private <T extends Oaf> void readActionsFromHDFSAndPromoteThemFor(String graphTableName,
|
||||
OafMergeAndGet.Strategy strategy,
|
||||
Class<T> clazz) throws Exception {
|
||||
// given
|
||||
// NOTE: test resource should contain atomic actions in a human readable form, probably as json files; here the
|
||||
// files should be converted to a serialized format and written out to workingDir/input
|
||||
// for current testing: actions from iis export, given as sequence file are copied to workingDir/input/
|
||||
|
||||
//graph
|
||||
Path inputGraphDir = inputDir.resolve("graph");
|
||||
Files.createDirectories(inputGraphDir);
|
||||
copyFiles(Paths.get(Objects.requireNonNull(cl.getResource("graph")).getFile()), inputGraphDir);
|
||||
|
||||
//actions
|
||||
Path inputActionsDir = inputDir.resolve("actions");
|
||||
Files.createDirectories(inputActionsDir);
|
||||
|
||||
Path inputEntitiesPatentDir = inputActionsDir.resolve("entities_patent");
|
||||
Files.createDirectories(inputEntitiesPatentDir);
|
||||
copyFiles(Paths.get(Objects.requireNonNull(cl.getResource("actions/entities_patent")).getFile()), inputEntitiesPatentDir);
|
||||
|
||||
Path inputEntitiesSoftwareDir = inputActionsDir.resolve("entities_software");
|
||||
Files.createDirectories(inputEntitiesSoftwareDir);
|
||||
copyFiles(Paths.get(Objects.requireNonNull(cl.getResource("actions/entities_software")).getFile()), inputEntitiesSoftwareDir);
|
||||
|
||||
String inputActionSetPaths = String.join(",", inputEntitiesSoftwareDir.toString()); //inputEntitiesPatentDir.toString(),
|
||||
String inputGraphTableJsonDumpPath =
|
||||
String.format("%s/%s.json", "eu/dnetlib/dhp/actionmanager/input/graph", graphTableName);
|
||||
createGraphTableFor(inputGraphTableJsonDumpPath, graphTableName, clazz);
|
||||
String inputActionSetPaths = createActionSets();
|
||||
Path outputGraphDir = outputDir.resolve("graph");
|
||||
|
||||
// when
|
||||
PromoteActionSetFromHDFSJob.main(new String[]{
|
||||
"-master", "local[*]",
|
||||
"-isSparkSessionManaged", Boolean.FALSE.toString(),
|
||||
"-inputGraphPath", inputGraphDir.toString(),
|
||||
"-inputActionSetPaths", inputActionSetPaths,
|
||||
"-outputGraphPath", outputDir.toString()
|
||||
"-graphTableName", graphTableName,
|
||||
"-outputGraphPath", outputGraphDir.toString(),
|
||||
"mergeAndGetStrategy", strategy.name()
|
||||
});
|
||||
|
||||
// then
|
||||
Path outputGraphTableDir = outputGraphDir.resolve(graphTableName);
|
||||
assertTrue(Files.exists(outputGraphDir));
|
||||
|
||||
List<T> outputGraphTableRows = readGraphTableFromParquet(outputGraphTableDir.toString(), clazz).collectAsList();
|
||||
outputGraphTableRows.sort(Comparator.comparingInt(Object::hashCode));
|
||||
|
||||
assertEquals(10, outputGraphTableRows.size());
|
||||
|
||||
String expectedOutputGraphTableJsonDumpPath =
|
||||
String.format("%s/%s/%s.json", "eu/dnetlib/dhp/actionmanager/output/graph", strategy.name().toLowerCase(), graphTableName);
|
||||
Path expectedOutputGraphTableJsonDumpFile = Paths
|
||||
.get(Objects.requireNonNull(cl.getResource(expectedOutputGraphTableJsonDumpPath)).getFile());
|
||||
List<T> expectedOutputGraphTableRows = readGraphTableFromJSON(
|
||||
expectedOutputGraphTableJsonDumpFile, clazz)
|
||||
.collectAsList();
|
||||
expectedOutputGraphTableRows.sort(Comparator.comparingInt(Object::hashCode));
|
||||
|
||||
assertArrayEquals(expectedOutputGraphTableRows.toArray(), outputGraphTableRows.toArray());
|
||||
}
|
||||
|
||||
private static void copyFiles(Path source, Path target) throws IOException {
|
||||
Files.list(source).forEach(f -> {
|
||||
try {
|
||||
if (Files.isDirectory(f)) {
|
||||
Path subTarget = Files.createDirectories(target.resolve(f.getFileName()));
|
||||
copyFiles(f, subTarget);
|
||||
} else {
|
||||
Files.copy(f, target.resolve(f.getFileName()));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
private <T extends Oaf> void createGraphTableFor(String inputGraphTableJsonDumpPath,
|
||||
String inputGraphTableDirRelativePath,
|
||||
Class<T> clazz) {
|
||||
Path inputGraphTableJsonDumpFile = Paths
|
||||
.get(Objects.requireNonNull(cl.getResource(inputGraphTableJsonDumpPath)).getFile());
|
||||
Dataset<T> inputGraphTableDS = readGraphTableFromJSON(inputGraphTableJsonDumpFile, clazz);
|
||||
Path inputGraphTableDir = inputGraphDir.resolve(inputGraphTableDirRelativePath);
|
||||
inputGraphTableDS
|
||||
.toJSON()
|
||||
.javaRDD()
|
||||
.mapToPair(json -> new Tuple2<>(new Text(clazz.getCanonicalName()), new Text(json)))
|
||||
.saveAsNewAPIHadoopFile(inputGraphTableDir.toString(),
|
||||
Text.class,
|
||||
Text.class,
|
||||
SequenceFileOutputFormat.class,
|
||||
configuration);
|
||||
}
|
||||
|
||||
private String createActionSets() throws IOException {
|
||||
Path inputActionPayloadJsonDumpsDir = Paths
|
||||
.get(Objects.requireNonNull(cl.getResource("eu/dnetlib/dhp/actionmanager/input/action_sets/"))
|
||||
.getFile());
|
||||
Files
|
||||
.list(inputActionPayloadJsonDumpsDir)
|
||||
.forEach(inputActionPayloadJsonDump -> {
|
||||
String inputActionSetId = inputActionPayloadJsonDump.getFileName().toString();
|
||||
Path inputActionSetDir = inputActionSetsDir.resolve(inputActionPayloadJsonDump.getFileName());
|
||||
spark.read()
|
||||
.textFile(inputActionPayloadJsonDump.toString())
|
||||
.withColumn("TargetValue", base64(col("value")))
|
||||
.select("TargetValue")
|
||||
.toJSON()
|
||||
.javaRDD()
|
||||
.mapToPair(json -> new Tuple2<>(new Text(inputActionSetId), new Text(json)))
|
||||
.saveAsNewAPIHadoopFile(inputActionSetDir.toString(),
|
||||
Text.class,
|
||||
Text.class,
|
||||
SequenceFileOutputFormat.class,
|
||||
configuration);
|
||||
});
|
||||
return Files.list(inputActionSetsDir).map(Path::toString).collect(Collectors.joining(","));
|
||||
}
|
||||
|
||||
private static <T extends Oaf> Dataset<T> readGraphTableFromJSON(Path path, Class<T> clazz) {
|
||||
return spark.read()
|
||||
.format("json")
|
||||
.load(path.toString())
|
||||
.toJSON()
|
||||
.map((MapFunction<String, T>) json -> newObjectMapper().readValue(json, clazz), Encoders.bean(clazz));
|
||||
}
|
||||
|
||||
//TODO: enable strict deserialization later
|
||||
private static ObjectMapper newObjectMapper() {
|
||||
return new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
|
||||
}
|
||||
|
||||
private static <T extends Oaf> Dataset<T> readGraphTableFromParquet(String outputGraphTablePath, Class<T> clazz) {
|
||||
return spark.read()
|
||||
.format("parquet")
|
||||
.load(outputGraphTablePath)
|
||||
.as(Encoders.bean(clazz));
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,20 @@
|
|||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::007a4870b31056f89b768cf508e1538e"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::007a4870b31056f89b768cf508e1538e"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::9ea9c0996c87e1dc7fc69f94b5ed0010"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::9ea9c0996c87e1dc7fc69f94b5ed0010"}
|
||||
{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::d0bbea1f5bed5864d1904eb602e608a6"}
|
||||
{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::d0bbea1f5bed5864d1904eb602e608a6"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","subRelType":"provision","target":"20|openaire____::c24a458004a31f9687089ea3d249de51"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","subRelType":"provision","target":"20|openaire____::c24a458004a31f9687089ea3d249de51"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::8de0f5a712997aafe0d794a53e51b75a"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::8de0f5a712997aafe0d794a53e51b75a"}
|
||||
{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::b58bdbe8ae5acead04fc76777d2f8017"}
|
||||
{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::b58bdbe8ae5acead04fc76777d2f8017"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|NARCIS__cris::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::c978e29d3b2ddf4f0c2b6e60d6613426"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|NARCIS__cris::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::c978e29d3b2ddf4f0c2b6e60d6613426"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|OpenstarTs__::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::fc7459b8fed8c0d47947fe04275251c0"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|OpenstarTs__::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::fc7459b8fed8c0d47947fe04275251c0"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|UnityFVG____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::89bab7c5a227fc27b2b9cadf475a6b71"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|UnityFVG____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::89bab7c5a227fc27b2b9cadf475a6b71"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|VTTRsInSsCrs::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::735915884eb439d42953372eaf934782"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|VTTRsInSsCrs::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::735915884eb439d42953372eaf934782"}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,10 @@
|
|||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::007a4870b31056f89b768cf508e1538e"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::9ea9c0996c87e1dc7fc69f94b5ed0010"}
|
||||
{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::d0bbea1f5bed5864d1904eb602e608a6"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","subRelType":"provision","target":"20|openaire____::c24a458004a31f9687089ea3d249de51"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::8de0f5a712997aafe0d794a53e51b75a"}
|
||||
{"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::b58bdbe8ae5acead04fc76777d2f8017"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|NARCIS__cris::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::c978e29d3b2ddf4f0c2b6e60d6613426"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|OpenstarTs__::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::fc7459b8fed8c0d47947fe04275251c0"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|UnityFVG____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::89bab7c5a227fc27b2b9cadf475a6b71"}
|
||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|VTTRsInSsCrs::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::735915884eb439d42953372eaf934782"}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,10 @@
|
|||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::d0bbea1f5bed5864d1904eb602e608a6"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|OpenstarTs__::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::fc7459b8fed8c0d47947fe04275251c0"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|NARCIS__cris::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::c978e29d3b2ddf4f0c2b6e60d6613426"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::b58bdbe8ae5acead04fc76777d2f8017"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|MetisRadboud::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::8de0f5a712997aafe0d794a53e51b75a"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|UnityFVG____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::89bab7c5a227fc27b2b9cadf475a6b71"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|4ScienceCRIS::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::007a4870b31056f89b768cf508e1538e"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|VTTRsInSsCrs::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|openaire____::735915884eb439d42953372eaf934782"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CRIS_UNS____::f66f1bd369679b5b077dcdf006089556","subRelType":"provision","target":"20|dedup_wf_001::9ea9c0996c87e1dc7fc69f94b5ed0010"}
|
||||
{"collectedFrom":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"lastupdatetimestamp":0,"relClass":"provides","relType":"datasourceOrganization","source":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","subRelType":"provision","target":"20|openaire____::c24a458004a31f9687089ea3d249de51"}
|
||||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue