workflow and parameters to exucute the dump

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json

@@ -1,4 +1,11 @@
+
 [
+	{
+		"paramName":"is",
+		"paramLongName":"isLookUpUrl",
+		"paramDescription": "URL of the isLookUp Service",
+		"paramRequired": true
+	},
 	{
 		"paramName":"s",
 		"paramLongName":"sourcePath",
@@ -16,5 +23,26 @@
 		"paramLongName": "isSparkSessionManaged",
 		"paramDescription": "true if the spark session is managed, false otherwise",
 		"paramRequired": false
+	},
+	{
+		"paramName":"tn",
+		"paramLongName":"resultTableName",
+		"paramDescription": "the name of the result table we are currently working on",
+		"paramRequired": true
+	},
+	{
+		"paramName":"dn",
+		"paramLongName":"dumpTableName",
+		"paramDescription": "the name of the corresondent dump element ",
+		"paramRequired": true
+	},
+	{
+		"paramName":"rt",
+		"paramLongName":"resultType",
+		"paramDescription": "the name of the corresondent dump element ",
+		"paramRequired": true
 	}
 ]
+
+
+

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml

@@ -1,10 +1,18 @@
 <workflow-app name="import_graph_as_hive_DB" xmlns="uri:oozie:workflow:0.5">
 
     <parameters>
-        <property>
-            <name>inputPath</name>
-            <description>the source path</description>
-        </property>
+            <property>
+                <name>sourcePath</name>
+                <description>the source path</description>
+            </property>
+            <property>
+                <name>isLookUpUrl</name>
+                <description>the isLookup service endpoint</description>
+            </property>
+            <property>
+                <name>outputPath</name>
+                <description>the output path</description>
+            </property>
         <property>
             <name>hiveDbName</name>
             <description>the target hive database name</description>
@@ -72,45 +80,35 @@
         </configuration>
     </global>
 
-    <start to="reset_DB"/>
+    <start to="reset_outputpath"/>
 
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
-    <action name="reset_DB">
-        <hive2 xmlns="uri:oozie:hive2-action:0.1">
-            <configuration>
-                <property>
-                    <name>hive.metastore.uris</name>
-                    <value>${hiveMetastoreUris}</value>
-                </property>
-            </configuration>
-            <jdbc-url>${hiveJdbcUrl}/${hiveDbName}</jdbc-url>
-            <script>lib/scripts/reset_db.sql</script>
-            <param>hiveDbName=${hiveDbName}</param>
-        </hive2>
-        <ok to="fork_import"/>
+    <action name="reset_outputpath">
+        <fs>
+            <delete path="${outputPath}"/>
+            <mkdir path="${outputPath}"/>
+        </fs>
+        <ok to="fork_dump"/>
         <error to="Kill"/>
     </action>
 
-    <fork name="fork_import">
-        <path start="import_publication"/>
-        <path start="import_dataset"/>
-        <path start="import_orp"/>
-        <path start="import_software"/>
-        <path start="import_datasource"/>
-        <path start="import_organization"/>
-        <path start="import_project"/>
-        <path start="import_relation"/>
+
+    <fork name="fork_dump">
+        <path start="dump_publication"/>
+        <path start="dump_dataset"/>
+        <path start="dump_orp"/>
+        <path start="dump_software"/>
     </fork>
 
-    <action name="import_publication">
+    <action name="dump_publication">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Import table publication</name>
-            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>
+            <name>Dump table publication for community related products</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkDumpCommunityProducts</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -122,21 +120,23 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}/publication</arg>
-            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
-            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
-            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--sourcePath</arg><arg>${inputPath}/publication</arg>
+            <arg>--inputType</arg><arg>publication</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
+            <arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/publication</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLoohUpUrl}</arg>
         </spark>
-        <ok to="join_import"/>
+        <ok to="join_dump"/>
         <error to="Kill"/>
     </action>
 
-    <action name="import_dataset">
+    <action name="dump_dataset">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Import table dataset</name>
-            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>
+            <name>Dump table dataset for community related products</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkDumpCommunityProducts</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -148,21 +148,23 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}/dataset</arg>
-            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
-            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
-            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--sourcePath</arg><arg>${inputPath}/dataset</arg>
+            <arg>--inputType</arg><arg>dataset</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
+            <arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/dataset</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLoohUpUrl}</arg>
         </spark>
-        <ok to="join_import"/>
+        <ok to="join_dump"/>
         <error to="Kill"/>
     </action>
 
-    <action name="import_orp">
+    <action name="dump_orp">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Import table otherresearchproduct</name>
-            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>
+            <name>Dump table ORP for community related products</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkDumpCommunityProducts</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -174,21 +176,23 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}/otherresearchproduct</arg>
-            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
-            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
-            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--sourcePath</arg><arg>${inputPath}/otherresearchproduct</arg>
+            <arg>--inputType</arg><arg>otherresearchproduct</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
+            <arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/otherresearchproduct</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLoohUpUrl}</arg>
         </spark>
-        <ok to="join_import"/>
+        <ok to="join_dump"/>
         <error to="Kill"/>
     </action>
 
-    <action name="import_software">
+    <action name="dump_software">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Import table software</name>
-            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>
+            <name>Dump table software for community related products</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkDumpCommunityProducts</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -200,21 +204,25 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}/software</arg>
-            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
-            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
-            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--sourcePath</arg><arg>${inputPath}/software</arg>
+            <arg>--inputType</arg><arg>software</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
+            <arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/software</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLoohUpUrl}</arg>
         </spark>
-        <ok to="join_import"/>
+        <ok to="join_dump"/>
         <error to="Kill"/>
     </action>
 
-    <action name="import_datasource">
+    <join name="join_dump" to="prepareResultProject"/>
+
+    <action name="prepareResultProject">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Import table datasource</name>
-            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>
+            <name>Prepare association result subset of project info</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkPrepareResultProject</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -226,21 +234,26 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}/datasource</arg>
-            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
-            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Datasource</arg>
-            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--sourcePath</arg><arg>${inputPath}</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
-        <ok to="join_import"/>
+        <ok to="fork_extendWithProject"/>
         <error to="Kill"/>
     </action>
 
-    <action name="import_organization">
+    <fork name="fork_extendWithProject">
+        <path start="extend_publication"/>
+        <path start="extend_dataset"/>
+        <path start="extend_orp"/>
+        <path start="extend_software"/>
+    </fork>
+
+    <action name="extend_publication">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Import table organization</name>
-            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>
+            <name>Extend dumped publications with information about projects</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkUpdateProjectInfo</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -252,21 +265,21 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}/organization</arg>
-            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
-            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Organization</arg>
-            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--sourcePath</arg><arg>${workingDir}/publication</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/ext/publication</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>
+            <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
-        <ok to="join_import"/>
+        <ok to="join_extend"/>
         <error to="Kill"/>
     </action>
 
-    <action name="import_project">
+    <action name="extend_dataset">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Import table project</name>
-            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>
+            <name>Extend dumped dataset with information about projects</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkUpdateProjectInfo</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -278,21 +291,20 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}/project</arg>
-            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
-            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Project</arg>
-            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--sourcePath</arg><arg>${workingDir}/dataset</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/ext/dataset</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>
+            <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
-        <ok to="join_import"/>
+        <ok to="join_extend"/>
         <error to="Kill"/>
     </action>
-
-    <action name="import_relation">
+    <action name="extend_orp">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Import table project</name>
-            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>
+            <name>Extend dumped ORP with information about projects</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkUpdateProjectInfo</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -304,33 +316,182 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}/relation</arg>
-            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
-            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Relation</arg>
-            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--sourcePath</arg><arg>${workingDir}/otherresearchproduct</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/ext/otherresearchproduct</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>
+            <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
-        <ok to="join_import"/>
+        <ok to="join_extend"/>
+        <error to="Kill"/>
+    </action>
+    <action name="extend_software">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Extend dumped software with information about projects</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkUpdateProjectInfo</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/software</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/ext/software</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
+            <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
+        </spark>
+        <ok to="join_extend"/>
         <error to="Kill"/>
     </action>
 
-    <join name="join_import" to="PostProcessing"/>
 
-    <action name="PostProcessing">
-        <hive2 xmlns="uri:oozie:hive2-action:0.1">
-            <configuration>
-                <property>
-                    <name>hive.metastore.uris</name>
-                    <value>${hiveMetastoreUris}</value>
-                </property>
-            </configuration>
-            <jdbc-url>${hiveJdbcUrl}/${hiveDbName}</jdbc-url>
-            <script>lib/scripts/postprocessing.sql</script>
-            <param>hiveDbName=${hiveDbName}</param>
-        </hive2>
-        <ok to="End"/>
+    <join name="join_extend" to="fork_splitForCommunities"/>
+
+
+    <fork name="fork_splitForCommunities">
+        <path start="split_publication"/>
+        <path start="split_dataset"/>
+        <path start="split_orp"/>
+        <path start="split_software"/>
+    </fork>
+
+    <action name="split_publication">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Split dumped result for community</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/ext/publication</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLoohUpUrl}</arg>
+        </spark>
+        <ok to="join_split"/>
         <error to="Kill"/>
     </action>
 
+    <action name="split_dataset">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Split dumped result for community</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/ext/dataset</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLoohUpUrl}</arg>
+        </spark>
+        <ok to="join_split"/>
+        <error to="Kill"/>
+    </action>
+    <action name="split_orp">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Split dumped result for community</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/ext/orp</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLoohUpUrl}</arg>
+        </spark>
+        <ok to="join_split"/>
+        <error to="Kill"/>
+    </action>
+    <action name="split_software">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Split dumped result for community</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/ext/software</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+            <arg>--className</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLoohUpUrl}</arg>
+        </spark>
+        <ok to="join_split"/>
+        <error to="Kill"/>
+    </action>
+
+<!--    <join name="join_split" to="loadInZenodo"/>-->
+    <join name="join_split" to="End"/>
+
+<!--    <action name="loadInZenodo">-->
+<!--        <spark xmlns="uri:oozie:spark-action:0.2">-->
+<!--            <master>yarn</master>-->
+<!--            <mode>cluster</mode>-->
+<!--            <name>Import table software</name>-->
+<!--            <class>eu.dnetlib.dhp.oa.graph.hive.GraphHiveTableImporterJob</class>-->
+<!--            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
+<!--            <spark-opts>-->
+<!--                &#45;&#45;executor-memory=${sparkExecutorMemory}-->
+<!--                &#45;&#45;executor-cores=${sparkExecutorCores}-->
+<!--                &#45;&#45;driver-memory=${sparkDriverMemory}-->
+<!--                &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
+<!--                &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
+<!--                &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
+<!--                &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
+<!--                &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
+<!--            </spark-opts>-->
+<!--            <arg>&#45;&#45;inputPath</arg><arg>${workingDir}/ext/publication</arg>-->
+<!--            <arg>&#45;&#45;hiveDbName</arg><arg>${hiveDbName}</arg>-->
+<!--            <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>-->
+<!--            -->
+<!--        </spark>-->
+<!--        <ok to="End"/>-->
+<!--        <error to="Kill"/>-->
+<!--    </action>-->
+
     <end name="End"/>
 
 </workflow-app>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/project_input_parameters.json

@@ -1,11 +1,6 @@
 
 [
-	{
-		"paramName":"is",
-		"paramLongName":"isLookUpUrl",
-		"paramDescription": "URL of the isLookUp Service",
-		"paramRequired": true
-	},
+
 	{
 		"paramName":"s",
 		"paramLongName":"sourcePath",
@@ -31,18 +26,11 @@
 		"paramRequired": true
 	},
 	{
-		"paramName":"dn",
-		"paramLongName":"dumpTableName",
-		"paramDescription": "the name of the corresondent dump element ",
-		"paramRequired": true
-	},
-	{
-		"paramName":"rt",
-		"paramLongName":"resultType",
-		"paramDescription": "the name of the corresondent dump element ",
+		"paramName": "pip",
+		"paramLongName": "preparedInfoPath",
+		"paramDescription": "the path of the association result projectlist",
 		"paramRequired": true
 	}
 ]
 
 
-

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/project_prepare_parameters.json

@@ -1,20 +1,20 @@
 [
 	{
-		"paramName": "issm",
+		"paramName":"s",
+		"paramLongName":"sourcePath",
+		"paramDescription": "the path of the sequencial file to read",
+		"paramRequired": true
+	},
+	{
+		"paramName": "out",
+		"paramLongName": "outputPath",
+		"paramDescription": "the path used to store temporary output files",
+		"paramRequired": true
+	},
+	{
+		"paramName": "ssm",
 		"paramLongName": "isSparkSessionManaged",
-		"paramDescription": "when true will stop SparkSession after job execution",
+		"paramDescription": "true if the spark session is managed, false otherwise",
 		"paramRequired": false
-	},
-	{
-		"paramName": "s",
-		"paramLongName": "sourcePath",
-		"paramDescription": "the source path",
-		"paramRequired": true
-	},
-	{
-		"paramName": "g",
-		"paramLongName": "graphRawPath",
-		"paramDescription": "the path of the graph Raw in hdfs",
-		"paramRequired": true
 	}
 ]

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json

@@ -29,18 +29,6 @@
 		"paramLongName":"resultTableName",
 		"paramDescription": "the name of the result table we are currently working on",
 		"paramRequired": true
-	},
-	{
-		"paramName":"dn",
-		"paramLongName":"dumpTableName",
-		"paramDescription": "the name of the corresondent dump element ",
-		"paramRequired": true
-	},
-	{
-		"paramName":"rt",
-		"paramLongName":"resultType",
-		"paramDescription": "the name of the corresondent dump element ",
-		"paramRequired": true
 	}
 ]