From 7b2a7e2417a15f87ad0df699346d0ab3a6862ff2 Mon Sep 17 00:00:00 2001 From: pjacewicz Date: Thu, 2 Apr 2020 11:48:40 +0200 Subject: [PATCH] [dhp-actionmanager] missing descriptions added and minor naming and formatting fixes --- .../PartitionActionSetsByPayloadTypeJob.java | 3 +- .../PromoteActionPayloadForGraphTableJob.java | 3 +- .../PromoteActionPayloadFunctions.java | 6 ++-- ...sets_by_payload_type_input_parameters.json | 8 ++--- ...load_for_graph_table_input_parameters.json | 14 ++++----- .../wf/dataset/oozie_app/workflow.xml | 26 ++++++++-------- .../wf/datasource/oozie_app/workflow.xml | 24 +++++++-------- .../wf/main/oozie_app/workflow.xml | 30 +++++++++---------- .../wf/organization/oozie_app/workflow.xml | 24 +++++++-------- .../oozie_app/workflow.xml | 26 ++++++++-------- .../wf/project/oozie_app/workflow.xml | 24 +++++++-------- .../wf/publication/oozie_app/workflow.xml | 26 ++++++++-------- .../wf/relation/oozie_app/workflow.xml | 24 +++++++-------- .../wf/software/oozie_app/workflow.xml | 26 ++++++++-------- 14 files changed, 133 insertions(+), 131 deletions(-) diff --git a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/partition/PartitionActionSetsByPayloadTypeJob.java b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/partition/PartitionActionSetsByPayloadTypeJob.java index 5b977c4ff..dbae0560b 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/partition/PartitionActionSetsByPayloadTypeJob.java +++ b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/partition/PartitionActionSetsByPayloadTypeJob.java @@ -73,7 +73,8 @@ public class PartitionActionSetsByPayloadTypeJob { }); } - private static void removeOutputDir(SparkSession spark, String path) { + private static void removeOutputDir(SparkSession spark, + String path) { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } diff --git a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadForGraphTableJob.java b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadForGraphTableJob.java index 21607a7dc..fc7f6d04e 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadForGraphTableJob.java +++ b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadForGraphTableJob.java @@ -120,7 +120,8 @@ public class PromoteActionPayloadForGraphTableJob { } } - private static void removeOutputDir(SparkSession spark, String path) { + private static void removeOutputDir(SparkSession spark, + String path) { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } diff --git a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadFunctions.java b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadFunctions.java index dce1c72a5..5773b6654 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadFunctions.java +++ b/dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/PromoteActionPayloadFunctions.java @@ -37,8 +37,8 @@ public class PromoteActionPayloadFunctions { * @param rowClazz Class of graph table * @param actionPayloadClazz Class of action payload * @param Type of graph table row - * @param Type of action payload - * @return Dataset of merged graph table rows and action payload + * @param Type of action payload instance + * @return Dataset of merged graph table rows and action payload instances */ public static Dataset joinGraphTableWithActionPayloadAndMerge(Dataset rowDS, Dataset actionPayloadDS, @@ -84,7 +84,7 @@ public class PromoteActionPayloadFunctions { * * @param rowDS Dataset representing graph table * @param rowIdFn Function used to get the id of graph table row - * @param mergeAndGetFn Function used to merge graph table row and action payload instance + * @param mergeAndGetFn Function used to merge graph table rows * @param zeroFn Function to create a zero/empty instance of graph table row * @param isNotZeroFn Function to check if graph table row is not zero/empty * @param rowClazz Class of graph table diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/partition/partition_action_sets_by_payload_type_input_parameters.json b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/partition/partition_action_sets_by_payload_type_input_parameters.json index 38630bc92..c2594ba49 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/partition/partition_action_sets_by_payload_type_input_parameters.json +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/partition/partition_action_sets_by_payload_type_input_parameters.json @@ -2,19 +2,19 @@ { "paramName": "issm", "paramLongName": "isSparkSessionManaged", - "paramDescription": "TODO", + "paramDescription": "when true will stop SparkSession after job execution", "paramRequired": false }, { "paramName": "iasp", "paramLongName": "inputActionSetPaths", - "paramDescription": "TODO", + "paramDescription": "comma separated list of action sets to partition by payload type", "paramRequired": true }, { - "paramName": "opasp", + "paramName": "op", "paramLongName": "outputPath", - "paramDescription": "TODO", + "paramDescription": "root output location for partitioned action sets", "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/promote/promote_action_payload_for_graph_table_input_parameters.json b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/promote/promote_action_payload_for_graph_table_input_parameters.json index 8da862573..e111f156e 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/promote/promote_action_payload_for_graph_table_input_parameters.json +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/promote/promote_action_payload_for_graph_table_input_parameters.json @@ -2,43 +2,43 @@ { "paramName": "issm", "paramLongName": "isSparkSessionManaged", - "paramDescription": "TODO", + "paramDescription": "when true will stop SparkSession after job execution", "paramRequired": false }, { "paramName": "igtp", "paramLongName": "inputGraphTablePath", - "paramDescription": "TODO", + "paramDescription": "location of graph table to promote", "paramRequired": true }, { "paramName": "gtcn", "paramLongName": "graphTableClassName", - "paramDescription": "TODO", + "paramDescription": "canonical name of graph table row class", "paramRequired": true }, { "paramName": "iapp", "paramLongName": "inputActionPayloadPath", - "paramDescription": "TODO", + "paramDescription": "location of action payload to promote", "paramRequired": true }, { "paramName": "apcn", "paramLongName": "actionPayloadClassName", - "paramDescription": "TODO", + "paramDescription": "canonical name of action payload class", "paramRequired": true }, { "paramName": "ogtp", "paramLongName": "outputGraphTablePath", - "paramDescription": "TODO", + "paramDescription": "location of promoted graph table", "paramRequired": true }, { "paramName": "mags", "paramLongName": "mergeAndGetStrategy", - "paramDescription": "TODO", + "paramDescription": "strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET", "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml index faee2b0bf..27c272aea 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml @@ -2,27 +2,27 @@ activePromoteDatasetActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Dataset payload activePromoteResultActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Result payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionPayloadRootPath - TODO + root location of action payloads to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory @@ -81,7 +81,7 @@ ${(activePromoteDatasetActionPayload eq "true") and - (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphPath')),'/'),'dataset')) eq "true") and + (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphRootPath')),'/'),'dataset')) eq "true") and (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputActionPayloadRootPath')),'/'),'clazz=eu.dnetlib.dhp.schema.oaf.Dataset')) eq "true")} @@ -105,7 +105,7 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=2560 - --inputGraphTablePath${inputGraphPath}/dataset + --inputGraphTablePath${inputGraphRootPath}/dataset --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Dataset --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Dataset --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Dataset @@ -119,7 +119,7 @@ -pb - ${inputGraphPath}/dataset + ${inputGraphRootPath}/dataset ${workingDir}/dataset @@ -157,7 +157,7 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Dataset --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Result --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Result - --outputGraphTablePath${outputGraphPath}/dataset + --outputGraphTablePath${outputGraphRootPath}/dataset --mergeAndGetStrategy${mergeAndGetStrategy} @@ -168,7 +168,7 @@ -pb ${workingDir}/dataset - ${outputGraphPath}/dataset + ${outputGraphRootPath}/dataset diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml index 545091a73..800f29b17 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml @@ -2,23 +2,23 @@ activePromoteDatasourceActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Datasource payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionPayloadRootPath - TODO + root location of action payloads to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory @@ -77,7 +77,7 @@ ${(activePromoteDatasourceActionPayload eq "true") and - (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphPath')),'/'),'datasource')) eq "true") and + (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphRootPath')),'/'),'datasource')) eq "true") and (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputActionPayloadRootPath')),'/'),'clazz=eu.dnetlib.dhp.schema.oaf.Datasource')) eq "true")} @@ -100,11 +100,11 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --inputGraphTablePath${inputGraphPath}/datasource + --inputGraphTablePath${inputGraphRootPath}/datasource --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Datasource --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Datasource --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Datasource - --outputGraphTablePath${outputGraphPath}/datasource + --outputGraphTablePath${outputGraphRootPath}/datasource --mergeAndGetStrategy${mergeAndGetStrategy} @@ -114,8 +114,8 @@ -pb - ${inputGraphPath}/datasource - ${outputGraphPath}/datasource + ${inputGraphRootPath}/datasource + ${outputGraphRootPath}/datasource diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml index 9150ebc06..4f54c0699 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml @@ -2,55 +2,55 @@ activePromoteDatasetActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Dataset payload activePromoteDatasourceActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Datasource payload activePromoteOrganizationActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Organization payload activePromoteOtherResearchProductActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.OtherResearchProduct payload activePromoteProjectActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Project payload activePromotePublicationActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Publication payload activePromoteRelationActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Relation payload activePromoteResultActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Result payload activePromoteSoftwareActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Software payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionSetPaths - TODO + comma separated list of action sets to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml index d67698ce5..a6c6220e4 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml @@ -2,23 +2,23 @@ activePromoteOrganizationActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Organization payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionPayloadRootPath - TODO + root location of action payloads to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory @@ -77,7 +77,7 @@ ${(activePromoteOrganizationActionPayload eq "true") and - (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphPath')),'/'),'organization')) eq "true") and + (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphRootPath')),'/'),'organization')) eq "true") and (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputActionPayloadRootPath')),'/'),'clazz=eu.dnetlib.dhp.schema.oaf.Organization')) eq "true")} @@ -100,11 +100,11 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --inputGraphTablePath${inputGraphPath}/organization + --inputGraphTablePath${inputGraphRootPath}/organization --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Organization --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Organization --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Organization - --outputGraphTablePath${outputGraphPath}/organization + --outputGraphTablePath${outputGraphRootPath}/organization --mergeAndGetStrategy${mergeAndGetStrategy} @@ -114,8 +114,8 @@ -pb - ${inputGraphPath}/organization - ${outputGraphPath}/organization + ${inputGraphRootPath}/organization + ${outputGraphRootPath}/organization diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml index 66b51ea75..4b65de3df 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml @@ -2,27 +2,27 @@ activePromoteOtherResearchProductActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.OtherResearchProduct payload activePromoteResultActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Result payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionPayloadRootPath - TODO + root location of action payloads to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory @@ -81,7 +81,7 @@ ${(activePromoteOtherResearchProductActionPayload eq "true") and - (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphPath')),'/'),'otherresearchproduct')) eq "true") and + (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphRootPath')),'/'),'otherresearchproduct')) eq "true") and (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputActionPayloadRootPath')),'/'),'clazz=eu.dnetlib.dhp.schema.oaf.OtherResearchProduct')) eq "true")} @@ -104,7 +104,7 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --inputGraphTablePath${inputGraphPath}/otherresearchproduct + --inputGraphTablePath${inputGraphRootPath}/otherresearchproduct --graphTableClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.OtherResearchProduct --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct @@ -118,7 +118,7 @@ -pb - ${inputGraphPath}/otherresearchproduct + ${inputGraphRootPath}/otherresearchproduct ${workingDir}/otherresearchproduct @@ -156,7 +156,7 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Result --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Result - --outputGraphTablePath${outputGraphPath}/otherresearchproduct + --outputGraphTablePath${outputGraphRootPath}/otherresearchproduct --mergeAndGetStrategy${mergeAndGetStrategy} @@ -167,7 +167,7 @@ -pb ${workingDir}/otherresearchproduct - ${outputGraphPath}/otherresearchproduct + ${outputGraphRootPath}/otherresearchproduct diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml index a84b35e14..c73fea96e 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml @@ -2,23 +2,23 @@ activePromoteProjectActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.OtherResearchProduct payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionPayloadRootPath - TODO + root location of action payloads to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory @@ -77,7 +77,7 @@ ${(activePromoteProjectActionPayload eq "true") and - (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphPath')),'/'),'project')) eq "true") and + (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphRootPath')),'/'),'project')) eq "true") and (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputActionPayloadRootPath')),'/'),'clazz=eu.dnetlib.dhp.schema.oaf.Project')) eq "true")} @@ -100,11 +100,11 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --inputGraphTablePath${inputGraphPath}/project + --inputGraphTablePath${inputGraphRootPath}/project --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Project --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Project --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Project - --outputGraphTablePath${outputGraphPath}/project + --outputGraphTablePath${outputGraphRootPath}/project --mergeAndGetStrategy${mergeAndGetStrategy} @@ -114,8 +114,8 @@ -pb - ${inputGraphPath}/project - ${outputGraphPath}/project + ${inputGraphRootPath}/project + ${outputGraphRootPath}/project diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml index 229909dd6..35be128da 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml @@ -2,27 +2,27 @@ activePromotePublicationActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Publication payload activePromoteResultActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Result payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionPayloadRootPath - TODO + root location of action payloads to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory @@ -81,7 +81,7 @@ ${(activePromotePublicationActionPayload eq "true") and - (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphPath')),'/'),'publication')) eq "true") and + (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphRootPath')),'/'),'publication')) eq "true") and (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputActionPayloadRootPath')),'/'),'clazz=eu.dnetlib.dhp.schema.oaf.Publication')) eq "true")} @@ -105,7 +105,7 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=2560 - --inputGraphTablePath${inputGraphPath}/publication + --inputGraphTablePath${inputGraphRootPath}/publication --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Publication --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Publication @@ -119,7 +119,7 @@ -pb - ${inputGraphPath}/publication + ${inputGraphRootPath}/publication ${workingDir}/publication @@ -157,7 +157,7 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Result --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Result - --outputGraphTablePath${outputGraphPath}/publication + --outputGraphTablePath${outputGraphRootPath}/publication --mergeAndGetStrategy${mergeAndGetStrategy} @@ -168,7 +168,7 @@ -pb ${workingDir}/publication - ${outputGraphPath}/publication + ${outputGraphRootPath}/publication diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml index 6059c024c..f64301c70 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml @@ -2,23 +2,23 @@ activePromoteRelationActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Relation payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionPayloadRootPath - TODO + root location of action payloads to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory @@ -77,7 +77,7 @@ ${(activePromoteRelationActionPayload eq "true") and - (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphPath')),'/'),'relation')) eq "true") and + (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphRootPath')),'/'),'relation')) eq "true") and (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputActionPayloadRootPath')),'/'),'clazz=eu.dnetlib.dhp.schema.oaf.Relation')) eq "true")} @@ -101,11 +101,11 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=2560 - --inputGraphTablePath${inputGraphPath}/relation + --inputGraphTablePath${inputGraphRootPath}/relation --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Relation --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Relation --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Relation - --outputGraphTablePath${outputGraphPath}/relation + --outputGraphTablePath${outputGraphRootPath}/relation --mergeAndGetStrategy${mergeAndGetStrategy} @@ -115,8 +115,8 @@ -pb - ${inputGraphPath}/relation - ${outputGraphPath}/relation + ${inputGraphRootPath}/relation + ${outputGraphRootPath}/relation diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml index 68167524d..397184c91 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml @@ -2,27 +2,27 @@ activePromoteSoftwareActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Software payload activePromoteResultActionPayload - TODO + when true will promote actions with eu.dnetlib.dhp.schema.oaf.Result payload - inputGraphPath - TODO + inputGraphRootPath + root location of input materialized graph inputActionPayloadRootPath - TODO + root location of action payloads to promote - outputGraphPath - TODO + outputGraphRootPath + root location for output materialized graph mergeAndGetStrategy - TODO + strategy for merging graph table objects with action payload instances, MERGE_FROM_AND_GET or SELECT_NEWER_AND_GET sparkDriverMemory @@ -81,7 +81,7 @@ ${(activePromoteSoftwareActionPayload eq "true") and - (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphPath')),'/'),'software')) eq "true") and + (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputGraphRootPath')),'/'),'software')) eq "true") and (fs:exists(concat(concat(concat(concat(wf:conf('nameNode'),'/'),wf:conf('inputActionPayloadRootPath')),'/'),'clazz=eu.dnetlib.dhp.schema.oaf.Software')) eq "true")} @@ -104,7 +104,7 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --inputGraphTablePath${inputGraphPath}/software + --inputGraphTablePath${inputGraphRootPath}/software --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Software --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Software --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Software @@ -118,7 +118,7 @@ -pb - ${inputGraphPath}/software + ${inputGraphRootPath}/software ${workingDir}/software @@ -156,7 +156,7 @@ --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Software --inputActionPayloadPath${inputActionPayloadRootPath}/clazz=eu.dnetlib.dhp.schema.oaf.Result --actionPayloadClassNameeu.dnetlib.dhp.schema.oaf.Result - --outputGraphTablePath${outputGraphPath}/software + --outputGraphTablePath${outputGraphRootPath}/software --mergeAndGetStrategy${mergeAndGetStrategy} @@ -167,7 +167,7 @@ -pb ${workingDir}/software - ${outputGraphPath}/software + ${outputGraphRootPath}/software