lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

minor

This commit is contained in:
Miriam Baglioni 2020-04-30 15:23:49 +02:00
parent 65a5d67b8b
commit 13f30664ea
2 changed files with 18 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob2.java
View File

@ -70,7 +70,7 @@ public class SparkResultToCommunityFromOrganizationJob2 {
if (isTest(parser)) {
removeOutputDir(spark, outputPath);
}
if(saveGraph)
if (saveGraph)
execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath);
});
}

37
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java
View File

@ -58,8 +58,6 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
final String resultClassName = parser.get("resultTableName");
log.info("resultTableName: {}", resultClassName);
final Boolean saveGraph = Optional
.ofNullable(parser.get("saveGraph"))
.map(Boolean::valueOf)
@ -78,7 +76,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
if (isTest(parser)) {
removeOutputDir(spark, outputPath);
}
if(saveGraph)
if (saveGraph)
execPropagation(
spark,
datasourceorganization,
@ -86,7 +84,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
inputPath,
outputPath,
resultClazz);
});
});
}
private static void execPropagation(
@ -112,22 +110,21 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
broadcast_datasourceorganizationassoc)
.as(Encoders.bean(ResultOrganizationSet.class));
getNewRelations(
spark
.read()
.textFile(alreadylinked)
.map(
value -> OBJECT_MAPPER
.readValue(
value, ResultOrganizationSet.class),
Encoders.bean(ResultOrganizationSet.class)),
potentialUpdates)
.toJSON()
.write()
.mode(SaveMode.Append)
.option("compression", "gzip")
.text(outputPath);
getNewRelations(
spark
.read()
.textFile(alreadylinked)
.map(
value -> OBJECT_MAPPER
.readValue(
value, ResultOrganizationSet.class),
Encoders.bean(ResultOrganizationSet.class)),
potentialUpdates)
.toJSON()
.write()
.mode(SaveMode.Append)
.option("compression", "gzip")
.text(outputPath);
}