D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minor fix

This commit is contained in:
Sandro La Bruzzo 2024-05-02 13:58:28 +02:00
parent c5ceaeb12b
commit ef104b10e6
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
airflow/dags/run_spark.py
View File

@ -36,6 +36,10 @@ from airflow import DAG
from airflow.providers.cncf.kubernetes.operators.spark_kubernetes import SparkKubernetesOperator from airflow.providers.cncf.kubernetes.operators.spark_kubernetes import SparkKubernetesOperator
from airflow.providers.cncf.kubernetes.sensors.spark_kubernetes import SparkKubernetesSensor from airflow.providers.cncf.kubernetes.sensors.spark_kubernetes import SparkKubernetesSensor
from airflow.utils.dates import days_ago from airflow.utils.dates import days_ago
import logging
logger = logging.getLogger(__name__)
# [END import_module] # [END import_module]
@ -64,6 +68,11 @@ spec =SparkConfigurator(
executor_memoryOverhead="3G" executor_memoryOverhead="3G"
).get_configuration() ).get_configuration()
logger.info("found configuration")
logger.info(spec)
dag = DAG( dag = DAG(
'spark_scholix', 'spark_scholix',
default_args=default_args, default_args=default_args,