alessia.bardi
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

moved check from join to method

This commit is contained in:
Miriam Baglioni 2020-05-11 14:11:44 +02:00
parent f5d785e096
commit e563e65335
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/SparkRemoveBlacklistedRelationJob.java
View File

@ -119,8 +119,7 @@ public class SparkRemoveBlacklistedRelationJob {
.and( .and(
inputRelation inputRelation
.col("target") .col("target")
.equalTo(dedupBL.col("target")) .equalTo(dedupBL.col("target")))),
.and(inputRelation.col("relclass").equalTo(dedupBL.col("relclass"))))),
"left_outer"); "left_outer");
log.info("numberOfRelationAfterJoin: {}", tmp.count()); log.info("numberOfRelationAfterJoin: {}", tmp.count());
@ -151,7 +150,8 @@ public class SparkRemoveBlacklistedRelationJob {
private static boolean areEquals(Relation ir, Relation bl) { private static boolean areEquals(Relation ir, Relation bl) {
return ir.getRelClass().equals(bl.getRelClass()) && return ir.getRelClass().equals(bl.getRelClass()) &&
ir.getRelType().equals(bl.getRelType()) && ir.getRelType().equals(bl.getRelType()) &&
ir.getSubRelType().equals(bl.getSubRelType()); ir.getSubRelType().equals(bl.getSubRelType()) &&
ir.getRelClass().equals(bl.getRelClass());
} }
public static org.apache.spark.sql.Dataset<Relation> readRelations( public static org.apache.spark.sql.Dataset<Relation> readRelations(