From 7f9b7cfcf651821d74b65dbb78981194f0cc29d3 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 7 Apr 2021 12:11:36 +0200 Subject: [PATCH] removing from the dump organization that have been deleted by inference --- .../dhp/oa/graph/dump/complete/DumpGraphEntities.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpGraphEntities.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpGraphEntities.java index d2bb269064..05f0d7def5 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpGraphEntities.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpGraphEntities.java @@ -9,6 +9,7 @@ import java.util.*; import java.util.stream.Collectors; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SaveMode; @@ -453,7 +454,7 @@ public class DumpGraphEntities implements Serializable { .map( (MapFunction) o -> mapOrganization((eu.dnetlib.dhp.schema.oaf.Organization) o), Encoders.bean(Organization.class)) - .filter(Objects::nonNull) + .filter((FilterFunction) o -> o != null) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") @@ -461,11 +462,10 @@ public class DumpGraphEntities implements Serializable { } private static Organization mapOrganization(eu.dnetlib.dhp.schema.oaf.Organization org) { + Organization organization = new Organization(); if (org.getDataInfo().getDeletedbyinference()) return null; - Organization organization = new Organization(); - Optional .ofNullable(org.getLegalshortname()) .ifPresent(value -> organization.setLegalshortname(value.getValue()));