From 1923c1ce21658c05e9822c9363a3509e4b63eb32 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 29 Jul 2021 11:36:20 +0200 Subject: [PATCH] replaced full join + filtering with a left join --- .../dnetlib/dhp/oa/graph/raw/PatchRelationsApplication.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/PatchRelationsApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/PatchRelationsApplication.java index 151f5ba3f..c2bcf69f0 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/PatchRelationsApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/PatchRelationsApplication.java @@ -80,8 +80,7 @@ public class PatchRelationsApplication { final Dataset idMapping = Utils.readPath(spark, idMappingPath, RelationIdMapping.class); rels - .joinWith(idMapping, rels.col("source").equalTo(idMapping.col("oldId")), "full") - .filter((FilterFunction>) t -> Objects.nonNull(t._1())) + .joinWith(idMapping, rels.col("source").equalTo(idMapping.col("oldId")), "left") .map((MapFunction, Relation>) t -> { final Relation r = t._1(); Optional.ofNullable(t._2()) @@ -89,8 +88,7 @@ public class PatchRelationsApplication { .ifPresent(r::setSource); return r; }, Encoders.bean(Relation.class)) - .joinWith(idMapping, rels.col("target").equalTo(idMapping.col("oldId")), "full") - .filter((FilterFunction>) t -> Objects.nonNull(t._1())) + .joinWith(idMapping, rels.col("target").equalTo(idMapping.col("oldId")), "left") .map((MapFunction, Relation>) t -> { final Relation r = t._1(); Optional.ofNullable(t._2())