D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minor fix

This commit is contained in:
Sandro La Bruzzo 2024-05-02 14:50:25 +02:00
parent 3666871e4d
commit cb06d46bb2
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
airflow/dags/run_spark.py
View File

@ -58,13 +58,13 @@ default_args = {
}
spec =SparkConfigurator(
name="spark-scholix", \
mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump", \
jarLocation = 's3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar'\
,arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"],\
executor_cores=10,\
executor_memory="4G", \
executor_instances=1, \
name="spark-scholix-{{ ds }}-{{ task_instance.try_number }}",
mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump",
jarLocation = 's3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar',
arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"],\
executor_cores=10,
executor_memory="4G",
executor_instances=1,
executor_memoryOverhead="3G").get_configuration()
logger.info("found configuration")
@ -86,7 +86,7 @@ submit = SparkKubernetesOperator(
kubernetes_conn_id="kubernetes_default",
# do_xcom_push=True,
# delete_on_termination=True,
base_container_name="spark-kubernetes-driver",
# base_container_name="spark-kubernetes-driver",
dag=dag
)