pispis
/
dnet-hadoop
forked from antonis.lempesis/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

bug fix: ids in self mergerels are not marked deletedbyinference=true

This commit is contained in:
miconis 2021-04-12 15:55:27 +02:00
parent 37b65cc3ad
commit dcff9cecdf
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkUpdateEntity.java
View File

@ -13,6 +13,7 @@ import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Encoders;
@ -91,6 +92,7 @@ public class SparkUpdateEntity extends AbstractSparkAction {
final JavaPairRDD<String, String> mergedIds = rel final JavaPairRDD<String, String> mergedIds = rel
.where("relClass == 'merges'") .where("relClass == 'merges'")
.where("source != target")
.select(rel.col("target")) .select(rel.col("target"))
.distinct() .distinct()
.toJavaRDD() .toJavaRDD()