From e43aeb139a685b8d9ef97a682318917232ab16ef Mon Sep 17 00:00:00 2001
From: "miriam.baglioni" <miriam.baglioni@isti.cnr.it>
Date: Mon, 3 Aug 2020 18:07:28 +0200
Subject: [PATCH] added new property file and changed some parameter to old
 files

---
 .../oa/graph/dump/input_cm_parameters.json    | 19 ++++-----------
 .../dhp/oa/graph/dump/input_parameters.json   |  7 +++++-
 .../dhp/oa/graph/dump/oozie_app/workflow.xml  | 24 +++++++++++++++----
 .../dhp/oa/graph/dump/split_parameters.json   | 12 ++++++++++
 .../dhp/oa/graph/dump/upload_zenodo.json      | 18 +++++++++-----
 5 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json
index b19be2f52..6e42bfa64 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json
@@ -1,11 +1,6 @@
 
 [
-	{
-		"paramName":"cmp",
-		"paramLongName":"communityMapPath",
-		"paramDescription": "the path to the serialization of the community map",
-		"paramRequired": false
-	},
+
 	{
 		"paramName":"is",
 		"paramLongName":"isLookUpUrl",
@@ -13,9 +8,9 @@
 		"paramRequired": true
 	},
 	{
-		"paramName":"s",
-		"paramLongName":"sourcePath",
-		"paramDescription": "the path of the sequencial file to read",
+		"paramName":"nn",
+		"paramLongName":"nameNode",
+		"paramDescription": "the name node",
 		"paramRequired": true
 	},
 	{
@@ -23,12 +18,6 @@
 		"paramLongName": "outputPath",
 		"paramDescription": "the path used to store temporary output files",
 		"paramRequired": true
-	},
-	{
-		"paramName": "ssm",
-		"paramLongName": "isSparkSessionManaged",
-		"paramDescription": "true if the spark session is managed, false otherwise",
-		"paramRequired": false
 	}
 ]
 
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json
index a1407ae67..0cc8c2b4c 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json
@@ -1,9 +1,14 @@
 [
-
 	{
 		"paramName":"is",
 		"paramLongName":"isLookUpUrl",
 		"paramDescription": "URL of the isLookUp Service",
+		"paramRequired": true
+	},
+	{
+		"paramName":"cmp",
+		"paramLongName":"communityMapPath",
+		"paramDescription": "the path to the serialization of the community map",
 		"paramRequired": false
 	},
 	{
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml
index 167c920b4..fc3debb64 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml
@@ -81,7 +81,7 @@
         </configuration>
     </global>
 
-    <start to="reset_outputpath"/>
+    <start to="save_community_map"/>
 
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
@@ -92,10 +92,20 @@
             <delete path="${outputPath}"/>
             <mkdir path="${outputPath}"/>
         </fs>
-        <ok to="fork_dump"/>
+        <ok to="save_community_map"/>
         <error to="Kill"/>
     </action>
 
+    <action name="save_community_map">
+        <java>
+            <main-class>eu.dnetlib.dhp.oa.graph.dump.community.SaveCommunityMap</main-class>
+            <arg>--outputPath</arg><arg>${workingDir}/communityMap</arg>
+            <arg>--nameNode</arg><arg>${nameNode}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+        </java>
+        <ok to="fork_dump"/>
+        <error to="Kill"/>
+    </action>
 
     <fork name="fork_dump">
         <path start="dump_publication"/>
@@ -124,6 +134,7 @@
             <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
             <arg>--outputPath</arg><arg>${workingDir}/publication</arg>
+            <arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
         <ok to="join_dump"/>
@@ -150,6 +161,7 @@
             <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
             <arg>--outputPath</arg><arg>${workingDir}/dataset</arg>
+            <arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
         <ok to="join_dump"/>
@@ -176,6 +188,7 @@
             <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
             <arg>--outputPath</arg><arg>${workingDir}/otherresearchproduct</arg>
+            <arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
         <ok to="join_dump"/>
@@ -202,6 +215,7 @@
             <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
             <arg>--outputPath</arg><arg>${workingDir}/software</arg>
+            <arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
         <ok to="join_dump"/>
@@ -346,7 +360,7 @@
                 <master>yarn</master>
                 <mode>cluster</mode>
                 <name>Split dumped result for community</name>
-                <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkSplitForCommunity</class>
+                <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkSplitForCommunityS3</class>
                 <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
                 <spark-opts>
                     --executor-memory=${sparkExecutorMemory}
@@ -360,9 +374,10 @@
                 </spark-opts>
                 <arg>--sourcePath</arg><arg>${workingDir}/ext</arg>
                 <arg>--outputPath</arg><arg>${outputPath}</arg>
+                <arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
                 <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
             </spark>
-            <ok to="send_zenodo"/>
+            <ok to="End"/>
             <error to="Kill"/>
         </action>
 
@@ -374,6 +389,7 @@
             <arg>--accessToken</arg><arg>${accessToken}</arg>
             <arg>--connectionUrl</arg><arg>${connectionUrl}</arg>
             <arg>--metadata</arg><arg>${metadata}</arg>
+            <arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </java>
         <ok to="End"/>
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json
index 60fa7a95f..36e72eb84 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json
@@ -1,5 +1,17 @@
 
 [
+	{
+		"paramName":"is",
+		"paramLongName":"isLookUpUrl",
+		"paramDescription": "URL of the isLookUp Service",
+		"paramRequired": true
+	},
+	{
+		"paramName":"cmp",
+		"paramLongName":"communityMapPath",
+		"paramDescription": "the path to the serialization of the community map",
+		"paramRequired": false
+	},
 	{
 		"paramName":"is",
 		"paramLongName":"isLookUpUrl",
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json
index d65df9584..0e527852b 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json
@@ -1,11 +1,17 @@
 
 [
-{
-"paramName":"is",
-"paramLongName":"isLookUpUrl",
-"paramDescription": "URL of the isLookUp Service",
-"paramRequired": true
-},
+  {
+    "paramName":"is",
+    "paramLongName":"isLookUpUrl",
+    "paramDescription": "URL of the isLookUp Service",
+    "paramRequired": true
+  },
+  {
+    "paramName":"cmp",
+    "paramLongName":"communityMapPath",
+    "paramDescription": "the path to the serialization of the community map",
+    "paramRequired": false
+  },
 {
 "paramName":"hdfsp",
 "paramLongName":"hdfsPath",