D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

moved check from join to method

This commit is contained in:
Miriam Baglioni 2020-05-11 14:11:44 +02:00
parent f5d785e096
commit e563e65335
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/SparkRemoveBlacklistedRelationJob.java
View File

@ -119,8 +119,7 @@ public class SparkRemoveBlacklistedRelationJob {
.and(
inputRelation
.col("target")
.equalTo(dedupBL.col("target"))
.and(inputRelation.col("relclass").equalTo(dedupBL.col("relclass"))))),
.equalTo(dedupBL.col("target")))),
"left_outer");
log.info("numberOfRelationAfterJoin: {}", tmp.count());
@ -151,7 +150,8 @@ public class SparkRemoveBlacklistedRelationJob {
private static boolean areEquals(Relation ir, Relation bl) {
return ir.getRelClass().equals(bl.getRelClass()) &&
ir.getRelType().equals(bl.getRelType()) &&
ir.getSubRelType().equals(bl.getSubRelType());
ir.getSubRelType().equals(bl.getSubRelType()) &&
ir.getRelClass().equals(bl.getRelClass());
}
public static org.apache.spark.sql.Dataset<Relation> readRelations(