D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

oct-update #1

Merged
sandro.labruzzo merged 42 commits from oct-update into master 2024-11-18 10:43:07 +01:00
Conversation 0 Commits 42 Files Changed 33 +5 -10
1 changed files with 5 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c6fbfd3f0a - Show all commits

15
workflow/dnet/dedup.py
View File

@ -113,8 +113,7 @@ createmergerel = SparkKubernetesOperator(
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup",
"--cutConnectedComponent", "200",
"--hiveMetastoreUris", "",
"--pivotHistoryDatabase", "",
"--numPartitions", "64"
"--pivotHistoryDatabase", ""
],
executor_cores=8,
executor_memory="4G",
@ -134,8 +133,7 @@ creatededuprecord = SparkKubernetesOperator(
arguments=["--graphBasePath", "s3a://graph/tmp/prod_provision/graph/05_graph_inferred",
"--isLookUpUrl", "http://services.openaire.eu:8280/is/services/isLookUp?wsdl",
"--actionSetId", "dedup-result-decisiontree-v4",
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup",
"--numPartitions", "64"
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup"
],
executor_cores=8,
executor_memory="4G",
@ -176,8 +174,7 @@ createorgsdeduprecord = SparkKubernetesOperator(
arguments=["--graphBasePath", "s3a://graph/tmp/prod_provision/graph/05_graph_inferred",
"--isLookUpUrl", "http://services.openaire.eu:8280/is/services/isLookUp?wsdl",
"--actionSetId", "dedup-result-decisiontree-v4",
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup",
"--numPartitions", "64"
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup"
],
executor_cores=8,
executor_memory="4G",
@ -196,8 +193,7 @@ updateentity = SparkKubernetesOperator(
jarLocation='s3a://binaries/dhp-shade-package-1.2.5-SNAPSHOT.jar',
arguments=["--graphBasePath", "s3a://graph/tmp/prod_provision/graph/05_graph_inferred",
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup",
"--dedupGraphPath", "s3a://graph/tmp/prod_provision/graph/06_graph_dedup",
"--numPartitions", "64"
"--dedupGraphPath", "s3a://graph/tmp/prod_provision/graph/06_graph_dedup"
],
executor_cores=8,
executor_memory="4G",
@ -216,8 +212,7 @@ copyrelations = SparkKubernetesOperator(
jarLocation='s3a://binaries/dhp-shade-package-1.2.5-SNAPSHOT.jar',
arguments=["--graphBasePath", "s3a://graph/tmp/prod_provision/graph/05_graph_inferred",
"--workingPath", "s3a://graph/tmp/prod_provision/working_dir/dedup",
"--dedupGraphPath", "s3a://graph/tmp/prod_provision/graph/06_graph_dedup",
"--numPartitions", "64"
"--dedupGraphPath", "s3a://graph/tmp/prod_provision/graph/06_graph_dedup"
],
executor_cores=8,
executor_memory="4G",