lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

using _the right_ scala 2.11 converters

This commit is contained in:
Claudio Atzori 2024-10-28 13:56:25 +01:00
parent e6ca382deb
commit 9b4415cb67
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateDedupRecord.java
View File

@ -26,6 +26,7 @@ import eu.dnetlib.dhp.utils.ISLookupClientFactory;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import eu.dnetlib.pace.config.DedupConfig;
import scala.collection.JavaConversions;
import scala.collection.JavaConverters;
public class SparkCreateDedupRecord extends AbstractSparkAction {
@ -98,9 +99,7 @@ public class SparkCreateDedupRecord extends AbstractSparkAction {
.read()
.load(mergeRelPath)
.where("relClass == 'merges'")
.join(
dedupIds, JavaConverters.asScalaBufferConverter(Arrays.asList("source", "target")).asScala(),
"left_semi")
.join(dedupIds, JavaConversions.asScalaBuffer(Arrays.asList("source", "target")), "left_semi")
.write()
.mode(SaveMode.Overwrite)
.option("compression", "gzip")