diff --git a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml index d930ab774..8b5313c15 100644 --- a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml @@ -46,6 +46,7 @@ create_openaire_ranking_graph.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -56,10 +57,20 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkHighDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba ${openaireDataInput} - + ${currentYear} ${sparkShufflePartitions} @@ -68,14 +79,14 @@ ${wfAppPath}/create_openaire_ranking_graph.py#create_openaire_ranking_graph.py - + - - - + + + @@ -83,7 +94,7 @@ - + @@ -96,12 +107,13 @@ yarn-cluster cluster - + Spark CC CC.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -112,6 +124,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkNormalDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba ${openaireGraphInputPath} @@ -119,13 +141,13 @@ ${wfAppPath}/CC.py#CC.py - + - - + + @@ -135,16 +157,17 @@ ${jobTracker} ${nameNode} - - + + yarn-cluster cluster - + Spark RAM TAR.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -155,6 +178,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkNormalDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba ${openaireGraphInputPath} ${ramGamma} @@ -166,17 +199,17 @@ ${wfAppPath}/TAR.py#TAR.py - + - - - + + + - + @@ -189,12 +222,13 @@ yarn-cluster cluster - + Spark Impulse CC.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -205,6 +239,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkNormalDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba ${openaireGraphInputPath} @@ -213,13 +257,13 @@ ${wfAppPath}/CC.py#CC.py - + - - + + @@ -234,7 +278,7 @@ ${jobTracker} ${nameNode} - + @@ -242,16 +286,17 @@ - - + + yarn-cluster cluster - + Spark Pagerank PageRank.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -262,6 +307,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkNormalDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba ${openaireGraphInputPath} ${pageRankAlpha} @@ -273,14 +328,14 @@ ${wfAppPath}/PageRank.py#PageRank.py - + - + - + @@ -289,16 +344,17 @@ ${jobTracker} ${nameNode} - - + + yarn-cluster cluster - + Spark AttRank AttRank.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -309,6 +365,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkNormalDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba ${openaireGraphInputPath} ${attrankAlpha} @@ -325,17 +391,17 @@ ${wfAppPath}/AttRank.py#AttRank.py - + - - - + + + - + @@ -345,35 +411,35 @@ ${jobTracker} ${nameNode} - + /usr/bin/bash get_ranking_files.sh /${workingDir} - + ${wfAppPath}/get_ranking_files.sh#get_ranking_files.sh - + - + - - + + - - + + @@ -383,16 +449,17 @@ ${jobTracker} ${nameNode} - + yarn-cluster cluster - + Format Ranking Results JSON format_ranking_results.py +<<<<<<< HEAD --executor-memory=${sparkNormalExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -403,6 +470,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkNormalExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkNormalDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba json-5-way @@ -417,14 +494,14 @@ openaire ${wfAppPath}/format_ranking_results.py#format_ranking_results.py - - + + - - + + @@ -433,16 +510,17 @@ ${jobTracker} ${nameNode} - + yarn-cluster cluster - + Format Ranking Results BiP! DB format_ranking_results.py +<<<<<<< HEAD --executor-memory=${sparkNormalExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -453,6 +531,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkNormalExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkNormalDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba zenodo @@ -467,16 +555,16 @@ openaire ${wfAppPath}/format_ranking_results.py#format_ranking_results.py - - + + - - - - + + + + @@ -490,16 +578,17 @@ - - + + yarn-cluster cluster - + Openaire-DOI synonym collection map_openaire_ids_to_dois.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -510,6 +599,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkHighDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba ${openaireDataInput} @@ -517,13 +616,13 @@ ${wfAppPath}/map_openaire_ids_to_dois.py#map_openaire_ids_to_dois.py - + - - + + @@ -535,15 +634,16 @@ ${nameNode} - + yarn-cluster cluster - + Mapping Openaire Scores to DOIs map_scores_to_dois.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} @@ -554,6 +654,16 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkHighDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba ${synonymFolder} @@ -568,13 +678,13 @@ ${wfAppPath}/map_scores_to_dois.py#map_scores_to_dois.py - + - - + + @@ -629,11 +739,18 @@ projects_impact.py +<<<<<<< HEAD --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} +======= + --executor-memory ${sparkHighExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory ${sparkNormalDriverMemory} + --master yarn + --deploy-mode cluster + --conf spark.sql.shuffle.partitions=7680 +>>>>>>> 4a905932a3db36c61570c24b9aa54283cd30abba --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}