D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

master #11

Manually merged
claudio.atzori merged 275 commits from :master into enrichment_wfs 2020-05-11 15:14:56 +02:00
Conversation 1 Commits 275 Files Changed 156 +7 -12
1 changed files with 7 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 5b3f8a0e90 - Show all commits

19
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/countrypropagation/SparkCountryPropagationJob3.java
View File

@ -71,20 +71,15 @@ public class SparkCountryPropagationJob3 {
Class<? extends Result> resultClazz = (Class<? extends Result>) Class.forName(resultClassName);
SparkConf conf = new SparkConf();
conf.registerKryoClasses(ModelSupport.getOafModelClasses());
runWithSparkSession(
conf,
isSparkSessionManaged,
spark -> {
execPropagation(
spark,
inputPath,
outputPath,
resultClazz,
saveGraph);
});
spark -> execPropagation(
spark,
inputPath,
outputPath,
resultClazz,
saveGraph));
}
private static <R extends Result> void execPropagation(
@ -101,7 +96,7 @@ public class SparkCountryPropagationJob3 {
spark
.read()
.json(inputPath)
.as(Encoders.kryo(resultClazz))
.as(Encoders.bean(resultClazz))
.groupByKey((MapFunction<R, String>) result1 -> result1.getId(), Encoders.STRING())
.mapGroups(getCountryMergeFn(resultClazz), Encoders.bean(resultClazz))
.write()