From a9120164aa8377d11ff6136d2be8d3c7bb6bf6cc Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 16 Apr 2020 18:42:04 +0200 Subject: [PATCH] added hive parameter and a step of reset of the working dir in the workflow --- .../oozie_app/config-default.xml | 4 ++ .../oozie_app/workflow.xml | 65 ++++++------------- 2 files changed, 24 insertions(+), 45 deletions(-) diff --git a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/config-default.xml b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/config-default.xml index 56d0ac43c8..8d2c341057 100644 --- a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/config-default.xml +++ b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/config-default.xml @@ -19,6 +19,10 @@ spark2YarnHistoryServerAddress http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + spark2EventLogDir /user/spark/spark2ApplicationHistory diff --git a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml index a3c043bf00..d20b839eea 100644 --- a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml @@ -17,13 +17,21 @@ writes new version of the graph after the propagation step + - + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - + + + + + + + @@ -35,7 +43,7 @@ yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-PreparePhase1-Publications eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1 dhp-propagation-${projectVersion}.jar @@ -63,7 +71,7 @@ yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-PreparePhase1-Dataset eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1 dhp-propagation-${projectVersion}.jar @@ -91,7 +99,7 @@ yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-PreparePhase1-ORP eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1 dhp-propagation-${projectVersion}.jar @@ -119,7 +127,7 @@ yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-PreparePhase1-Software eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1 dhp-propagation-${projectVersion}.jar @@ -139,50 +147,17 @@ --outputPath${workingDir}/preparedInfo/targetOrcidAssoc --allowedsemrels${allowedsemrels} - + - - - - - ${jobTracker} - ${nameNode} - yarn-cluster - cluster - OrcidToResultFromSemRelPropagation - eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob - dhp-propagation-${projectVersion}.jar - - --num-executors=${sparkExecutorNumber} - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - - -mt yarn-cluster - --sourcePath${sourcePath} - --allowedsemrels${allowedsemrels} - --writeUpdate${writeUpdate} - --saveGraph${saveGraph} - - - - - yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-PreparePhase2 eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep2 dhp-propagation-${projectVersion}.jar @@ -215,7 +190,7 @@ yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-Publication eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3 dhp-propagation-${projectVersion}.jar @@ -241,7 +216,7 @@ yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-Dataset eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3 dhp-propagation-${projectVersion}.jar @@ -267,7 +242,7 @@ yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-ORP eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3 dhp-propagation-${projectVersion}.jar @@ -293,7 +268,7 @@ yarn cluster - ORCIDpropagationFromSemRel + ORCIDPropagation-Software eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3 dhp-propagation-${projectVersion}.jar