diff --git a/airflow/dags/run_spark.py b/airflow/dags/run_spark.py index 5b31b3b..0e7d4ef 100644 --- a/airflow/dags/run_spark.py +++ b/airflow/dags/run_spark.py @@ -36,9 +36,6 @@ from airflow import DAG from airflow.providers.cncf.kubernetes.operators.spark_kubernetes import SparkKubernetesOperator from airflow.providers.cncf.kubernetes.sensors.spark_kubernetes import SparkKubernetesSensor from airflow.utils.dates import days_ago -import logging - -logger = logging.getLogger(__name__) # [END import_module] @@ -67,11 +64,6 @@ spec =SparkConfigurator( executor_instances=1, executor_memoryOverhead="3G").get_configuration() -logger.info("found configuration") - -logger.info(spec) - - dag = DAG( 'spark_scholix', default_args=default_args,