diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_all_steps/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_all_steps/oozie_app/workflow.xml index 8d7959110..42ab59822 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_all_steps/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_all_steps/oozie_app/workflow.xml @@ -1,4 +1,5 @@ + workingPath @@ -48,6 +49,21 @@ + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + + @@ -73,8 +89,6 @@ - ${jobTracker} - ${nameNode} eu.dnetlib.dhp.migration.step1.MigrateDbEntitiesApplication -p${workingPath}/db_records -pgurl${postgresURL} @@ -87,8 +101,6 @@ - ${jobTracker} - ${nameNode} eu.dnetlib.dhp.migration.step1.MigrateMongoMdstoresApplication -p${workingPath}/odf_records -mongourl${mongoURL} @@ -103,8 +115,6 @@ - ${jobTracker} - ${nameNode} eu.dnetlib.dhp.migration.step1.MigrateMongoMdstoresApplication -p${workingPath}/oaf_records -mongourl${mongoURL} @@ -127,9 +137,7 @@ - ${jobTracker} - ${nameNode} - yarn-cluster + yarn cluster GenerateEntities eu.dnetlib.dhp.migration.step2.GenerateEntitiesApplication @@ -164,9 +172,7 @@ - ${jobTracker} - ${nameNode} - yarn-cluster + yarn cluster GenerateGraph eu.dnetlib.dhp.migration.step3.DispatchEntitiesApplication diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/hive/postprocessing.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/lib/scripts/postprocessing.sql similarity index 100% rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/hive/postprocessing.sql rename to dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/lib/scripts/postprocessing.sql diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml index 7a883db4c..481cc70b4 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml @@ -1,4 +1,5 @@ + sourcePath @@ -22,6 +23,21 @@ + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + + @@ -30,9 +46,7 @@ - ${jobTracker} - ${nameNode} - yarn-cluster + yarn cluster MapGraphIntoDataFrame eu.dnetlib.dhp.graph.SparkGraphImporterJob @@ -56,9 +70,13 @@ - ${jobTracker} - ${nameNode} - + + + oozie.hive.defaults + hive-site.xml + + + hive_db_name=${hive_db_name} diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml index 3132ae940..b154b61e1 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/graph/oozie_app/workflow.xml @@ -1,4 +1,5 @@ + hive_db_name @@ -26,6 +27,21 @@ + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + + @@ -42,8 +58,6 @@ - ${jobTracker} - ${nameNode} yarn cluster build_adjacency_lists @@ -71,8 +85,6 @@ - ${jobTracker} - ${nameNode} yarn cluster to_solr_index