D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update spark-version

This commit is contained in:
Giambattista Bloisi 2024-10-19 11:09:24 +02:00
parent ba3f351736
commit c5f4263061
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
workflow/dnet/spark_configurator.py
View File

@ -70,7 +70,7 @@ class SparkConfigurator:
"mainClass": self.mainClass, "mainClass": self.mainClass,
"mainApplicationFile": self.jarLocation, "mainApplicationFile": self.jarLocation,
"arguments": self.arguments, "arguments": self.arguments,
"sparkVersion": "3.5.1", "sparkVersion": "3.5.3",
"sparkConf": self.s3Configuration, "sparkConf": self.s3Configuration,
"restartPolicy": { "restartPolicy": {
"type": "Never" "type": "Never"
@ -81,7 +81,7 @@ class SparkConfigurator:
"coreLimit": "1200m", "coreLimit": "1200m",
"memory": self.sparkResoruceConf['driver_memory'], "memory": self.sparkResoruceConf['driver_memory'],
"labels": { "labels": {
"version": "3.5.1" "version": "3.5.3"
}, },
"serviceAccount": "spark" "serviceAccount": "spark"
}, },
@ -92,7 +92,7 @@ class SparkConfigurator:
"memory": self.sparkResoruceConf['executor_memory'], "memory": self.sparkResoruceConf['executor_memory'],
"instances": self.sparkResoruceConf['executor_instances'], "instances": self.sparkResoruceConf['executor_instances'],
"labels": { "labels": {
"version": "3.5.1" "version": "3.5.3"
} }
} }
} }