D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed workflow parameters

This commit is contained in:
sandro.labruzzo 2024-11-18 16:42:16 +01:00
parent 4372ab9e1c
commit 9682e964b3
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
workflow/dnet/build_scholexplorer_graph.py
View File

@ -6,6 +6,7 @@ from airflow.models.param import Param
from airflow.providers.cncf.kubernetes.operators.spark_kubernetes import SparkKubernetesOperator
from spark_configurator import SparkConfigurator
from dag_utils import SPARK_RESOURCES_PROFILES
EXECUTION_TIMEOUT = int(os.getenv("EXECUTION_TIMEOUT", 6))
@ -36,9 +37,10 @@ def build_scholexplorer_dag():
template_spec=SparkConfigurator(
name="orcidpropagate-{{ ds }}-{{ task_instance.try_number }}",
mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump",
profile=SPARK_RESOURCES_PROFILES['medium'],
jarLocation='s3a://binaries/dhp-shade-package-1.2.5-SNAPSHOT.jar',
arguments=["--sourcePath", "{{ dag_run.conf.get('ORCID_PATH') }}",
"--targetPath", "{{ dag_run.conf.get('INPUT_PATH') }}",
arguments=["--sourcePath", "{{ dag_run.conf.get('INPUT_PATH') }}",
"--targetPath", "{{ dag_run.conf.get('OUTPUT_PATH') }}",
]).get_configuration(),
kubernetes_conn_id="kubernetes_default"
)