diff --git a/airflow/dags/run_spark.py b/airflow/dags/run_spark.py index 61a32b2..0edd99b 100644 --- a/airflow/dags/run_spark.py +++ b/airflow/dags/run_spark.py @@ -58,15 +58,14 @@ default_args = { } spec =SparkConfigurator( - name="spark-scholix", - mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump", - jarLocation="s3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar", - arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"], - executor_cores=10, - executor_memory="3G", - executor_instances=1, - executor_memoryOverhead="3G" -).get_configuration() + name="spark-scholix", \ + mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump", \ + jarLocation = 's3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar'\ + ,arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"],\ + executor_cores=10,\ + executor_memory="4G", \ + executor_instances=1, \ + executor_memoryOverhead="3G").get_configuration() logger.info("found configuration") diff --git a/airflow/dags/spark_configurator.py b/airflow/dags/spark_configurator.py index c2794f5..a4bf2d2 100644 --- a/airflow/dags/spark_configurator.py +++ b/airflow/dags/spark_configurator.py @@ -2,7 +2,7 @@ class SparkConfigurator: def __init__(self, name, mainClass, - jarLocation, + jarLocation:str, arguments, apiVersion=None, namespace="dnet-spark-jobs", @@ -22,7 +22,7 @@ class SparkConfigurator: self.name = name self.image= image self.mainClass = mainClass - self.jarLocation = jarLocation, + self.jarLocation = jarLocation self.arguments= arguments self.s3Configuration = { "spark.driver.extraJavaOptions": "-Divy.cache.dir=/tmp -Dcom.amazonaws.sdk.disableCertChecking=true -Dcom.cloudera.com.amazonaws.sdk.disableCertChecking=true", @@ -68,7 +68,7 @@ class SparkConfigurator: "image":self.image, "imagePullPolicy": "IfNotPresent", "mainClass": self.mainClass, - "mainApplicationFile": self.mainClass, + "mainApplicationFile": self.jarLocation, "arguments": self.arguments, "sparkVersion": "3.5.1", "sparkConf": self.s3Configuration,