diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json
index b19be2f52..6e42bfa64 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json
@@ -1,11 +1,6 @@
[
- {
- "paramName":"cmp",
- "paramLongName":"communityMapPath",
- "paramDescription": "the path to the serialization of the community map",
- "paramRequired": false
- },
+
{
"paramName":"is",
"paramLongName":"isLookUpUrl",
@@ -13,9 +8,9 @@
"paramRequired": true
},
{
- "paramName":"s",
- "paramLongName":"sourcePath",
- "paramDescription": "the path of the sequencial file to read",
+ "paramName":"nn",
+ "paramLongName":"nameNode",
+ "paramDescription": "the name node",
"paramRequired": true
},
{
@@ -23,12 +18,6 @@
"paramLongName": "outputPath",
"paramDescription": "the path used to store temporary output files",
"paramRequired": true
- },
- {
- "paramName": "ssm",
- "paramLongName": "isSparkSessionManaged",
- "paramDescription": "true if the spark session is managed, false otherwise",
- "paramRequired": false
}
]
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json
index a1407ae67..0cc8c2b4c 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json
@@ -1,9 +1,14 @@
[
-
{
"paramName":"is",
"paramLongName":"isLookUpUrl",
"paramDescription": "URL of the isLookUp Service",
+ "paramRequired": true
+ },
+ {
+ "paramName":"cmp",
+ "paramLongName":"communityMapPath",
+ "paramDescription": "the path to the serialization of the community map",
"paramRequired": false
},
{
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml
index 167c920b4..fc3debb64 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml
@@ -81,7 +81,7 @@
-
+
Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
@@ -92,10 +92,20 @@
-
+
+
+
+ eu.dnetlib.dhp.oa.graph.dump.community.SaveCommunityMap
+ --outputPath${workingDir}/communityMap
+ --nameNode${nameNode}
+ --isLookUpUrl${isLookUpUrl}
+
+
+
+
@@ -124,6 +134,7 @@
--sourcePath${sourcePath}/publication
--resultTableNameeu.dnetlib.dhp.schema.oaf.Publication
--outputPath${workingDir}/publication
+ --communityMapPath${workingDir}/communityMap
--isLookUpUrl${isLookUpUrl}
@@ -150,6 +161,7 @@
--sourcePath${sourcePath}/dataset
--resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset
--outputPath${workingDir}/dataset
+ --communityMapPath${workingDir}/communityMap
--isLookUpUrl${isLookUpUrl}
@@ -176,6 +188,7 @@
--sourcePath${sourcePath}/otherresearchproduct
--resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct
--outputPath${workingDir}/otherresearchproduct
+ --communityMapPath${workingDir}/communityMap
--isLookUpUrl${isLookUpUrl}
@@ -202,6 +215,7 @@
--sourcePath${sourcePath}/software
--resultTableNameeu.dnetlib.dhp.schema.oaf.Software
--outputPath${workingDir}/software
+ --communityMapPath${workingDir}/communityMap
--isLookUpUrl${isLookUpUrl}
@@ -346,7 +360,7 @@
yarn
cluster
Split dumped result for community
- eu.dnetlib.dhp.oa.graph.dump.community.SparkSplitForCommunity
+ eu.dnetlib.dhp.oa.graph.dump.community.SparkSplitForCommunityS3
dhp-graph-mapper-${projectVersion}.jar
--executor-memory=${sparkExecutorMemory}
@@ -360,9 +374,10 @@
--sourcePath${workingDir}/ext
--outputPath${outputPath}
+ --communityMapPath${workingDir}/communityMap
--isLookUpUrl${isLookUpUrl}
-
+
@@ -374,6 +389,7 @@
--accessToken${accessToken}
--connectionUrl${connectionUrl}
--metadata${metadata}
+ --communityMapPath${workingDir}/communityMap
--isLookUpUrl${isLookUpUrl}
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json
index 60fa7a95f..36e72eb84 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json
@@ -1,5 +1,17 @@
[
+ {
+ "paramName":"is",
+ "paramLongName":"isLookUpUrl",
+ "paramDescription": "URL of the isLookUp Service",
+ "paramRequired": true
+ },
+ {
+ "paramName":"cmp",
+ "paramLongName":"communityMapPath",
+ "paramDescription": "the path to the serialization of the community map",
+ "paramRequired": false
+ },
{
"paramName":"is",
"paramLongName":"isLookUpUrl",
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json
index d65df9584..0e527852b 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json
@@ -1,11 +1,17 @@
[
-{
-"paramName":"is",
-"paramLongName":"isLookUpUrl",
-"paramDescription": "URL of the isLookUp Service",
-"paramRequired": true
-},
+ {
+ "paramName":"is",
+ "paramLongName":"isLookUpUrl",
+ "paramDescription": "URL of the isLookUp Service",
+ "paramRequired": true
+ },
+ {
+ "paramName":"cmp",
+ "paramLongName":"communityMapPath",
+ "paramDescription": "the path to the serialization of the community map",
+ "paramRequired": false
+ },
{
"paramName":"hdfsp",
"paramLongName":"hdfsPath",