miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

minor modifications

This commit is contained in:
Miriam Baglioni 2020-04-30 14:45:27 +02:00
parent 418595fec2
commit 65a5d67b8b
2 changed files with 15 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob2.java
View File

@ -70,6 +70,7 @@ public class SparkResultToCommunityFromOrganizationJob2 {
if (isTest(parser)) { if (isTest(parser)) {
removeOutputDir(spark, outputPath); removeOutputDir(spark, outputPath);
} }
if(saveGraph)
execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath); execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath);
}); });
} }

17
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java
View File

@ -58,8 +58,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
final String resultClassName = parser.get("resultTableName"); final String resultClassName = parser.get("resultTableName");
log.info("resultTableName: {}", resultClassName); log.info("resultTableName: {}", resultClassName);
final String resultType = resultClassName.substring(resultClassName.lastIndexOf(".") + 1).toLowerCase();
log.info("resultType: {}", resultType);
final Boolean saveGraph = Optional final Boolean saveGraph = Optional
.ofNullable(parser.get("saveGraph")) .ofNullable(parser.get("saveGraph"))
@ -79,15 +78,14 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
if (isTest(parser)) { if (isTest(parser)) {
removeOutputDir(spark, outputPath); removeOutputDir(spark, outputPath);
} }
if(saveGraph)
execPropagation( execPropagation(
spark, spark,
datasourceorganization, datasourceorganization,
alreadylinked, alreadylinked,
inputPath, inputPath,
outputPath, outputPath,
resultClazz, resultClazz);
resultType,
saveGraph);
}); });
} }
@ -97,10 +95,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
String alreadylinked, String alreadylinked,
String inputPath, String inputPath,
String outputPath, String outputPath,
Class<? extends Result> resultClazz, Class<? extends Result> resultClazz) {
String resultType,
Boolean saveGraph) {
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
org.apache.spark.sql.Dataset<DatasourceOrganization> datasourceorganizationassoc = readAssocDatasourceOrganization( org.apache.spark.sql.Dataset<DatasourceOrganization> datasourceorganizationassoc = readAssocDatasourceOrganization(
@ -117,7 +112,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
broadcast_datasourceorganizationassoc) broadcast_datasourceorganizationassoc)
.as(Encoders.bean(ResultOrganizationSet.class)); .as(Encoders.bean(ResultOrganizationSet.class));
if (saveGraph) {
getNewRelations( getNewRelations(
spark spark
.read() .read()
@ -133,7 +128,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
.mode(SaveMode.Append) .mode(SaveMode.Append)
.option("compression", "gzip") .option("compression", "gzip")
.text(outputPath); .text(outputPath);
}
} }
private static Dataset<Relation> getNewRelations( private static Dataset<Relation> getNewRelations(