lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

using scala 2.11 converters

This commit is contained in:
Claudio Atzori 2024-10-28 13:52:06 +01:00
parent 940735921f
commit e6ca382deb
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateDedupRecord.java
View File

@ -98,7 +98,9 @@ public class SparkCreateDedupRecord extends AbstractSparkAction {
.read() .read()
.load(mergeRelPath) .load(mergeRelPath)
.where("relClass == 'merges'") .where("relClass == 'merges'")
.join(dedupIds, JavaConverters.asScalaBuffer(Arrays.asList("source", "target")).toSeq(), "left_semi") .join(
dedupIds, JavaConverters.asScalaBufferConverter(Arrays.asList("source", "target")).asScala(),
"left_semi")
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")