D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 10 Projects 2 Releases 2 Wiki Activity

-

This commit is contained in:
Miriam Baglioni 2020-05-07 11:47:11 +02:00
parent 9f8855991c e07feb4c5f
commit e124278934
2 changed files with 2 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
:
View File

@ -1,7 +0,0 @@
Merge remote-tracking branch 'upstream/master'
# Please enter a commit message to explain why this merge is necessary,
# especially if it merges an updated upstream into a topic branch.
#
# Lines starting with '#' will be ignored, and an empty message aborts
# the commit.

3
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/countrypropagation/SparkCountryPropagationJob3.java
View File

@ -71,6 +71,7 @@ public class SparkCountryPropagationJob3 {
Class<? extends Result> resultClazz = (Class<? extends Result>) Class.forName(resultClassName); Class<? extends Result> resultClazz = (Class<? extends Result>) Class.forName(resultClassName);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
conf.registerKryoClasses(ModelSupport.getOafModelClasses());
runWithSparkSession( runWithSparkSession(
conf, conf,
@ -100,7 +101,7 @@ public class SparkCountryPropagationJob3 {
spark spark
.read() .read()
.json(inputPath) .json(inputPath)
.as(Encoders.bean(resultClazz)) .as(Encoders.kryo(resultClazz))
.groupByKey((MapFunction<R, String>) result1 -> result1.getId(), Encoders.STRING()) .groupByKey((MapFunction<R, String>) result1 -> result1.getId(), Encoders.STRING())
.mapGroups(getCountryMergeFn(resultClazz), Encoders.bean(resultClazz)) .mapGroups(getCountryMergeFn(resultClazz), Encoders.bean(resultClazz))
.write() .write()