diff --git a/workflow/dnet/clean.py b/workflow/dnet/clean.py index 828ff07..c7d69fb 100644 --- a/workflow/dnet/clean.py +++ b/workflow/dnet/clean.py @@ -77,7 +77,7 @@ def clean_graph_dag(): for entity in dag_utils.GRAPH_ENTITIES: clean_tasks.append(SparkKubernetesOperator( task_id='cleansparkjob_' + entity, - task_display_name="MasterDuplicateAction", + task_display_name="Clean " + entity, namespace='dnet-spark-jobs', template_spec=SparkConfigurator( name="cleansparkjob-{{ ds }}-{{ task_instance.try_number }}", @@ -101,7 +101,9 @@ def clean_graph_dag(): kubernetes_conn_id="kubernetes_default" )) - chain(getdatasourcefromcountry, masterduplicateaction, clean_tasks) + chain(getdatasourcefromcountry, + #masterduplicateaction, + clean_tasks) clean_graph_dag()