D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

oct-update #1

Merged
sandro.labruzzo merged 42 commits from oct-update into master 2024-11-18 10:43:07 +01:00
Conversation 0 Commits 42 Files Changed 33 +2 -2
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 0a2956d81f - Show all commits

2
workflow/dnet/dedup.py
View File

@ -72,7 +72,7 @@ submit = SparkKubernetesOperator(
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup",
"--numPartitions", "64"
],
executor_cores=10,
executor_cores=4,
executor_memory="4G",
executor_instances=1,
executor_memoryOverhead="3G").get_configuration(),

2
workflow/dnet/run_spark.py
View File

@ -59,7 +59,7 @@ spec =SparkConfigurator(
mainClass="eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump",
jarLocation = 's3a://deps/dhp-shade-package-1.2.5-SNAPSHOT.jar',
arguments =[ "--sourcePath", "s3a://raw-graph/01", "--targetPath", "s3a://scholix"],\
executor_cores=10,
executor_cores=4,
executor_memory="4G",
executor_instances=1,
executor_memoryOverhead="3G").get_configuration()