D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minor fix

This commit is contained in:
Sandro La Bruzzo 2024-05-02 14:50:25 +02:00
parent 3666871e4d
commit cb06d46bb2
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
airflow/dags/run_spark.py
View File

@ -58,13 +58,13 @@ default_args = {
} }
spec =SparkConfigurator( spec =SparkConfigurator(
name="spark-scholix", \ name="spark-scholix-{{ ds }}-{{ task_instance.try_number }}",
mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump", \ mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump",
jarLocation = 's3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar'\ jarLocation = 's3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar',
,arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"],\ arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"],\
executor_cores=10,\ executor_cores=10,
executor_memory="4G", \ executor_memory="4G",
executor_instances=1, \ executor_instances=1,
executor_memoryOverhead="3G").get_configuration() executor_memoryOverhead="3G").get_configuration()
logger.info("found configuration") logger.info("found configuration")
@ -86,7 +86,7 @@ submit = SparkKubernetesOperator(
kubernetes_conn_id="kubernetes_default", kubernetes_conn_id="kubernetes_default",
# do_xcom_push=True, # do_xcom_push=True,
# delete_on_termination=True, # delete_on_termination=True,
base_container_name="spark-kubernetes-driver", # base_container_name="spark-kubernetes-driver",
dag=dag dag=dag
) )