minor modifications

This commit is contained in:
Miriam Baglioni 2020-04-30 14:45:27 +02:00
parent 418595fec2
commit 65a5d67b8b
2 changed files with 15 additions and 19 deletions

View File

@ -70,6 +70,7 @@ public class SparkResultToCommunityFromOrganizationJob2 {
if (isTest(parser)) {
removeOutputDir(spark, outputPath);
}
if(saveGraph)
execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath);
});
}

View File

@ -58,8 +58,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
final String resultClassName = parser.get("resultTableName");
log.info("resultTableName: {}", resultClassName);
final String resultType = resultClassName.substring(resultClassName.lastIndexOf(".") + 1).toLowerCase();
log.info("resultType: {}", resultType);
final Boolean saveGraph = Optional
.ofNullable(parser.get("saveGraph"))
@ -79,15 +78,14 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
if (isTest(parser)) {
removeOutputDir(spark, outputPath);
}
if(saveGraph)
execPropagation(
spark,
datasourceorganization,
alreadylinked,
inputPath,
outputPath,
resultClazz,
resultType,
saveGraph);
resultClazz);
});
}
@ -97,10 +95,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
String alreadylinked,
String inputPath,
String outputPath,
Class<? extends Result> resultClazz,
String resultType,
Boolean saveGraph) {
Class<? extends Result> resultClazz) {
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
org.apache.spark.sql.Dataset<DatasourceOrganization> datasourceorganizationassoc = readAssocDatasourceOrganization(
@ -117,7 +112,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
broadcast_datasourceorganizationassoc)
.as(Encoders.bean(ResultOrganizationSet.class));
if (saveGraph) {
getNewRelations(
spark
.read()
@ -133,7 +128,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
.mode(SaveMode.Append)
.option("compression", "gzip")
.text(outputPath);
}
}
private static Dataset<Relation> getNewRelations(