From 95cd2b9b1e7f130d53b616773e9e016ccf970b83 Mon Sep 17 00:00:00 2001
From: Giambattista Bloisi <gbloisi@gmail.com>
Date: Thu, 10 Aug 2023 11:53:48 +0200
Subject: [PATCH] Make filterInvisible a mandatory parameter of
 DispathEntitiesSparkJob Make filterInvisible a mandatory parameter of both
 dedup/consistency and graph/group oozie workflows

---
 .../eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java  | 6 +-----
 .../dhp/oa/dedup/consistency/oozie_app/workflow.xml        | 7 +++++--
 .../eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml   | 6 +++++-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java
index 88c32fdbf..4d2ccb178 100644
--- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java
+++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java
@@ -47,11 +47,7 @@ public class DispatchEntitiesSparkJob {
 		String outputPath = parser.get("outputPath");
 		log.info("outputPath: {}", outputPath);
 
-		boolean filterInvisible = Optional
-			.ofNullable(parser.get("filterInvisible"))
-			.map(Boolean::valueOf)
-			.orElse(Boolean.FALSE);
-
+		boolean filterInvisible = Boolean.valueOf(parser.get("filterInvisible"));
 		log.info("filterInvisible: {}", filterInvisible);
 
 		SparkConf conf = new SparkConf();
diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml
index b1ceca05a..eb7e6d98d 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml
@@ -12,6 +12,10 @@
             <name>graphOutputPath</name>
             <description>path of the output graph</description>
         </property>
+        <property>
+            <name>filterInvisible</name>
+            <description>whether filter out invisible entities after merge</description>
+        </property>
         <property>
             <name>sparkDriverMemory</name>
             <description>memory for driver process</description>
@@ -24,7 +28,6 @@
             <name>sparkExecutorCores</name>
             <description>number of cores used by single executor</description>
         </property>
-
         <property>
             <name>oozieActionShareLibForSpark2</name>
             <description>oozie action sharelib for spark 2.*</description>
@@ -167,7 +170,7 @@
             </spark-opts>
             <arg>--inputPath</arg><arg>${workingPath}/grouped_entities</arg>
             <arg>--outputPath</arg><arg>${graphOutputPath}</arg>
-            <arg>--filterInvisible</arg><arg>true</arg>
+            <arg>--filterInvisible</arg><arg>${filterInvisible}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml
index 27a18207b..219dc7331 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml
@@ -12,6 +12,10 @@
             <name>graphOutputPath</name>
             <description>path of the output graph</description>
         </property>
+        <property>
+            <name>filterInvisible</name>
+            <description>whether filter out invisible entities after merge</description>
+        </property>
         <property>
             <name>sparkDriverMemory</name>
             <description>memory for driver process</description>
@@ -117,7 +121,7 @@
             </spark-opts>
             <arg>--inputPath</arg><arg>${workingPath}/grouped_entities</arg>
             <arg>--outputPath</arg><arg>${graphOutputPath}</arg>
-            <arg>--filterInvisible</arg><arg>false</arg>
+            <arg>--filterInvisible</arg><arg>${filterInvisible}</arg>
         </spark>
         <ok to="delete_target_relation"/>
         <error to="Kill"/>