D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

master #11

Manually merged
claudio.atzori merged 275 commits from :master into enrichment_wfs 2020-05-11 15:14:56 +02:00
Conversation 1 Commits 275 Files Changed 156 +15 -19
2 changed files with 15 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 65a5d67b8b - Show all commits

3
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob2.java
View File

@ -70,7 +70,8 @@ public class SparkResultToCommunityFromOrganizationJob2 {
if (isTest(parser)) {
removeOutputDir(spark, outputPath);
}
execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath);
if(saveGraph)
execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath);
});
}

31
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java
View File

@ -58,8 +58,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
final String resultClassName = parser.get("resultTableName");
log.info("resultTableName: {}", resultClassName);
final String resultType = resultClassName.substring(resultClassName.lastIndexOf(".") + 1).toLowerCase();
log.info("resultType: {}", resultType);
final Boolean saveGraph = Optional
.ofNullable(parser.get("saveGraph"))
@ -79,16 +78,15 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
if (isTest(parser)) {
removeOutputDir(spark, outputPath);
}
execPropagation(
spark,
datasourceorganization,
alreadylinked,
inputPath,
outputPath,
resultClazz,
resultType,
saveGraph);
});
if(saveGraph)
execPropagation(
spark,
datasourceorganization,
alreadylinked,
inputPath,
outputPath,
resultClazz);
});
}
private static void execPropagation(
@ -97,10 +95,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
String alreadylinked,
String inputPath,
String outputPath,
Class<? extends Result> resultClazz,
String resultType,
Boolean saveGraph) {
Class<? extends Result> resultClazz) {
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
org.apache.spark.sql.Dataset<DatasourceOrganization> datasourceorganizationassoc = readAssocDatasourceOrganization(
@ -117,7 +112,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
broadcast_datasourceorganizationassoc)
.as(Encoders.bean(ResultOrganizationSet.class));
if (saveGraph) {
getNewRelations(
spark
.read()
@ -133,7 +128,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
.mode(SaveMode.Append)
.option("compression", "gzip")
.text(outputPath);
}
}
private static Dataset<Relation> getNewRelations(