D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reduce executor cores

This commit is contained in:
Giambattista Bloisi 2024-10-19 17:55:53 +02:00
parent 48f688cda9
commit 0a2956d81f
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
workflow/dnet/dedup.py
View File

@ -72,7 +72,7 @@ submit = SparkKubernetesOperator(
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup", "--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup",
"--numPartitions", "64" "--numPartitions", "64"
], ],
executor_cores=10, executor_cores=4,
executor_memory="4G", executor_memory="4G",
executor_instances=1, executor_instances=1,
executor_memoryOverhead="3G").get_configuration(), executor_memoryOverhead="3G").get_configuration(),

2
workflow/dnet/run_spark.py
View File

@ -59,7 +59,7 @@ spec =SparkConfigurator(
mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump", mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump",
jarLocation = 's3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar', jarLocation = 's3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar',
arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"],\ arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"],\
executor_cores=10, executor_cores=4,
executor_memory="4G", executor_memory="4G",
executor_instances=1, executor_instances=1,
executor_memoryOverhead="3G").get_configuration() executor_memoryOverhead="3G").get_configuration()