Clean DAG

workflow/dnet/clean.py

@@ -77,7 +77,7 @@ def clean_graph_dag():
     for entity in dag_utils.GRAPH_ENTITIES:
         clean_tasks.append(SparkKubernetesOperator(
             task_id='cleansparkjob_' + entity,
-            task_display_name="MasterDuplicateAction",
+            task_display_name="Clean " + entity,
             namespace='dnet-spark-jobs',
             template_spec=SparkConfigurator(
                 name="cleansparkjob-{{ ds }}-{{ task_instance.try_number }}",
@@ -101,7 +101,9 @@ def clean_graph_dag():
             kubernetes_conn_id="kubernetes_default"
         ))
 
-    chain(getdatasourcefromcountry, masterduplicateaction, clean_tasks)
+    chain(getdatasourcefromcountry,
+          #masterduplicateaction,
+          clean_tasks)
 
 
 clean_graph_dag()