From 65a5d67b8b543cb4287e85e4e16568e96c6b857e Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 30 Apr 2020 14:45:27 +0200 Subject: [PATCH] minor modifications --- ...ResultToCommunityFromOrganizationJob2.java | 3 +- ...rkResultToOrganizationFromIstRepoJob2.java | 31 ++++++++----------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob2.java b/dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob2.java index 74931a537..30908012b 100644 --- a/dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob2.java +++ b/dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob2.java @@ -70,7 +70,8 @@ public class SparkResultToCommunityFromOrganizationJob2 { if (isTest(parser)) { removeOutputDir(spark, outputPath); } - execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath); + if(saveGraph) + execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath); }); } diff --git a/dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java b/dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java index 3d1603c7e..b69158c87 100644 --- a/dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java +++ b/dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java @@ -58,8 +58,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 { final String resultClassName = parser.get("resultTableName"); log.info("resultTableName: {}", resultClassName); - final String resultType = resultClassName.substring(resultClassName.lastIndexOf(".") + 1).toLowerCase(); - log.info("resultType: {}", resultType); + final Boolean saveGraph = Optional .ofNullable(parser.get("saveGraph")) @@ -79,16 +78,15 @@ public class SparkResultToOrganizationFromIstRepoJob2 { if (isTest(parser)) { removeOutputDir(spark, outputPath); } - execPropagation( - spark, - datasourceorganization, - alreadylinked, - inputPath, - outputPath, - resultClazz, - resultType, - saveGraph); - }); + if(saveGraph) + execPropagation( + spark, + datasourceorganization, + alreadylinked, + inputPath, + outputPath, + resultClazz); + }); } private static void execPropagation( @@ -97,10 +95,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 { String alreadylinked, String inputPath, String outputPath, - Class resultClazz, - String resultType, - - Boolean saveGraph) { + Class resultClazz) { final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); org.apache.spark.sql.Dataset datasourceorganizationassoc = readAssocDatasourceOrganization( @@ -117,7 +112,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 { broadcast_datasourceorganizationassoc) .as(Encoders.bean(ResultOrganizationSet.class)); - if (saveGraph) { + getNewRelations( spark .read() @@ -133,7 +128,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 { .mode(SaveMode.Append) .option("compression", "gzip") .text(outputPath); - } + } private static Dataset getNewRelations(