1
0
Fork 0

Make filterInvisible a mandatory parameter of DispathEntitiesSparkJob

Make filterInvisible a mandatory parameter of both dedup/consistency and graph/group oozie workflows
This commit is contained in:
Giambattista Bloisi 2023-08-10 11:53:48 +02:00
parent fab9920271
commit 95cd2b9b1e
3 changed files with 11 additions and 8 deletions

View File

@ -47,11 +47,7 @@ public class DispatchEntitiesSparkJob {
String outputPath = parser.get("outputPath"); String outputPath = parser.get("outputPath");
log.info("outputPath: {}", outputPath); log.info("outputPath: {}", outputPath);
boolean filterInvisible = Optional boolean filterInvisible = Boolean.valueOf(parser.get("filterInvisible"));
.ofNullable(parser.get("filterInvisible"))
.map(Boolean::valueOf)
.orElse(Boolean.FALSE);
log.info("filterInvisible: {}", filterInvisible); log.info("filterInvisible: {}", filterInvisible);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();

View File

@ -12,6 +12,10 @@
<name>graphOutputPath</name> <name>graphOutputPath</name>
<description>path of the output graph</description> <description>path of the output graph</description>
</property> </property>
<property>
<name>filterInvisible</name>
<description>whether filter out invisible entities after merge</description>
</property>
<property> <property>
<name>sparkDriverMemory</name> <name>sparkDriverMemory</name>
<description>memory for driver process</description> <description>memory for driver process</description>
@ -24,7 +28,6 @@
<name>sparkExecutorCores</name> <name>sparkExecutorCores</name>
<description>number of cores used by single executor</description> <description>number of cores used by single executor</description>
</property> </property>
<property> <property>
<name>oozieActionShareLibForSpark2</name> <name>oozieActionShareLibForSpark2</name>
<description>oozie action sharelib for spark 2.*</description> <description>oozie action sharelib for spark 2.*</description>
@ -167,7 +170,7 @@
</spark-opts> </spark-opts>
<arg>--inputPath</arg><arg>${workingPath}/grouped_entities</arg> <arg>--inputPath</arg><arg>${workingPath}/grouped_entities</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}</arg> <arg>--outputPath</arg><arg>${graphOutputPath}</arg>
<arg>--filterInvisible</arg><arg>true</arg> <arg>--filterInvisible</arg><arg>${filterInvisible}</arg>
</spark> </spark>
<ok to="End"/> <ok to="End"/>
<error to="Kill"/> <error to="Kill"/>

View File

@ -12,6 +12,10 @@
<name>graphOutputPath</name> <name>graphOutputPath</name>
<description>path of the output graph</description> <description>path of the output graph</description>
</property> </property>
<property>
<name>filterInvisible</name>
<description>whether filter out invisible entities after merge</description>
</property>
<property> <property>
<name>sparkDriverMemory</name> <name>sparkDriverMemory</name>
<description>memory for driver process</description> <description>memory for driver process</description>
@ -117,7 +121,7 @@
</spark-opts> </spark-opts>
<arg>--inputPath</arg><arg>${workingPath}/grouped_entities</arg> <arg>--inputPath</arg><arg>${workingPath}/grouped_entities</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}</arg> <arg>--outputPath</arg><arg>${graphOutputPath}</arg>
<arg>--filterInvisible</arg><arg>false</arg> <arg>--filterInvisible</arg><arg>${filterInvisible}</arg>
</spark> </spark>
<ok to="delete_target_relation"/> <ok to="delete_target_relation"/>
<error to="Kill"/> <error to="Kill"/>