From ee84db7a6a5ff1e271f5d6a0cfdfcc07e1a59ac9 Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Fri, 6 Dec 2024 12:20:13 +0100 Subject: [PATCH] [communityfromsemrelpropagation] added filtering to remove the deletedbyinference and invisible results --- .../PrepareResultCommunitySetStep1.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java index 5af2bf4812..764390442d 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java @@ -70,7 +70,7 @@ public class PrepareResultCommunitySetStep1 { private static final String RESULT_WITH_CONTEXT = "select id, collect_set(co.id) community_context \n" + " from result " + " lateral view explode (context) c as co " + - " where datainfo.deletedbyinference = false AND lower(co.id) IN %s" + + " where lower(co.id) IN %s" + " group by id"; private static final String RESULT_PATENT = "select id " + @@ -160,7 +160,8 @@ public class PrepareResultCommunitySetStep1 { Dataset relation = readPath(spark, inputRelationPath, Relation.class); relation.createOrReplaceTempView("relation"); - Dataset result = readPath(spark, inputResultPath, resultClazz); + Dataset result = readPath(spark, inputResultPath, resultClazz) + .where("datainfo.deletedbyinference != true AND datainfo.invisible != true"); result.createOrReplaceTempView("result"); final String outputResultPath = outputPath + "/" + resultType;