D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

master #11

Manually merged
claudio.atzori merged 275 commits from :master into enrichment_wfs 2020-05-11 15:14:56 +02:00
Conversation 1 Commits 275 Files Changed 156 +5 -21
1 changed files with 5 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 757bae53ea - Show all commits

26
dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/SparkRemoveBlacklistedRelationJob.java
View File

@ -80,8 +80,6 @@ public class SparkRemoveBlacklistedRelationJob {
log.info("InputRelationCount: {}", inputRelation.count());
log.info("NumberOfBlacklistedRelations: {}", blackListed.count());
Dataset<Relation> dedupSource = blackListed
.joinWith(
mergesRelation, blackListed.col("source").equalTo(mergesRelation.col("target")),
@ -109,9 +107,8 @@ public class SparkRemoveBlacklistedRelationJob {
.mode(SaveMode.Overwrite)
.json(blacklistPath + "/deduped");
log.info("number of dedupedBL: {}", dedupBL.count());
Dataset<Tuple2<Relation, Relation>> tmp = inputRelation
inputRelation
.joinWith(
dedupBL, (inputRelation
.col("source")
@ -120,26 +117,19 @@ public class SparkRemoveBlacklistedRelationJob {
inputRelation
.col("target")
.equalTo(dedupBL.col("target")))),
"left_outer");
log.info("numberOfRelationAfterJoin: {}", tmp.count());
Dataset<Relation> tmp1 = tmp.map(c -> {
"left_outer")
.map(c -> {
Relation ir = c._1();
Optional<Relation> obl = Optional.ofNullable(c._2());
if (obl.isPresent()) {
if (areEquals(ir, obl.get())) {
if (ir.equals(obl.get())) {
return null;
}
}
return ir;
}, Encoders.bean(Relation.class))
.filter(Objects::nonNull);
log.info("NumberOfRelationAfterBlacklisting: {} ", tmp1.count());
tmp1
.filter(Objects::nonNull)
.write()
.mode(SaveMode.Overwrite)
.option("compression", "gzip")
@ -147,12 +137,6 @@ public class SparkRemoveBlacklistedRelationJob {
}
private static boolean areEquals(Relation ir, Relation bl) {
return ir.getRelClass().equals(bl.getRelClass()) &&
ir.getRelType().equals(bl.getRelType()) &&
ir.getSubRelType().equals(bl.getSubRelType()) &&
ir.getRelClass().equals(bl.getRelClass());
}
public static org.apache.spark.sql.Dataset<Relation> readRelations(
SparkSession spark, String inputPath) {