changed the input params and the workflow definition to tackle the Result as all result product produced

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json

@@ -36,18 +36,6 @@
 		"paramLongName":"resultTableName",
 		"paramDescription": "the name of the result table we are currently working on",
 		"paramRequired": true
-	},
-	{
-		"paramName":"dn",
-		"paramLongName":"dumpTableName",
-		"paramDescription": "the name of the corresondent dump element ",
-		"paramRequired": true
-	},
-	{
-		"paramName":"rt",
-		"paramLongName":"resultType",
-		"paramDescription": "the name of the corresondent dump element ",
-		"paramRequired": false
 	}
 ]
 

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml

@@ -121,9 +121,7 @@
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
-            <arg>--resultType</arg><arg>publication</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
-            <arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>
             <arg>--outputPath</arg><arg>${workingDir}/publication</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
@@ -149,9 +147,7 @@
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
-            <arg>--resultType</arg><arg>dataset</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
-            <arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>
             <arg>--outputPath</arg><arg>${workingDir}/dataset</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
@@ -177,9 +173,7 @@
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
-            <arg>--resultType</arg><arg>otherresearchproduct</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
-            <arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>
             <arg>--outputPath</arg><arg>${workingDir}/otherresearchproduct</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
@@ -205,9 +199,7 @@
                 --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
-            <arg>--resultType</arg><arg>software</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
-            <arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
             <arg>--outputPath</arg><arg>${workingDir}/software</arg>
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
         </spark>
@@ -267,7 +259,6 @@
             </spark-opts>
             <arg>--sourcePath</arg><arg>${workingDir}/publication</arg>
             <arg>--outputPath</arg><arg>${workingDir}/ext/publication</arg>
-            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>
             <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
         <ok to="join_extend"/>
@@ -293,7 +284,6 @@
             </spark-opts>
             <arg>--sourcePath</arg><arg>${workingDir}/dataset</arg>
             <arg>--outputPath</arg><arg>${workingDir}/ext/dataset</arg>
-            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>
             <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
         <ok to="join_extend"/>
@@ -318,7 +308,6 @@
             </spark-opts>
             <arg>--sourcePath</arg><arg>${workingDir}/otherresearchproduct</arg>
             <arg>--outputPath</arg><arg>${workingDir}/ext/orp</arg>
-            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>
             <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
         <ok to="join_extend"/>
@@ -343,25 +332,15 @@
             </spark-opts>
             <arg>--sourcePath</arg><arg>${workingDir}/software</arg>
             <arg>--outputPath</arg><arg>${workingDir}/ext/software</arg>
-            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
             <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
         <ok to="join_extend"/>
         <error to="Kill"/>
     </action>
 
+    <join name="join_extend" to="splitForCommunities"/>
 
-    <join name="join_extend" to="fork_splitForCommunities"/>
+        <action name="splitForCommunities">
-
-
-    <fork name="fork_splitForCommunities">
-        <path start="split_publication"/>
-        <path start="split_dataset"/>
-        <path start="split_orp"/>
-        <path start="split_software"/>
-    </fork>
-
-    <action name="split_publication">
             <spark xmlns="uri:oozie:spark-action:0.2">
                 <master>yarn</master>
                 <mode>cluster</mode>
@@ -378,93 +357,129 @@
                     --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                     --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
                 </spark-opts>
-            <arg>--sourcePath</arg><arg>${workingDir}/ext/publication</arg>
+                <arg>--sourcePath</arg><arg>${workingDir}/ext</arg>
                 <arg>--outputPath</arg><arg>${outputPath}</arg>
-            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>
+<!--                <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>-->
                 <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
             </spark>
-        <ok to="join_split"/>
+            <ok to="End"/>
             <error to="Kill"/>
         </action>
 
-    <action name="split_dataset">
+<!--    <join name="join_extend" to="fork_splitForCommunities"/>-->
-        <spark xmlns="uri:oozie:spark-action:0.2">
+
-            <master>yarn</master>
+
-            <mode>cluster</mode>
+<!--    <fork name="fork_splitForCommunities">-->
-            <name>Split dumped result for community</name>
+<!--        <path start="split_publication"/>-->
-            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>
+<!--        <path start="split_dataset"/>-->
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+<!--        <path start="split_orp"/>-->
-            <spark-opts>
+<!--        <path start="split_software"/>-->
-                --executor-memory=${sparkExecutorMemory}
+<!--    </fork>-->
-                --executor-cores=${sparkExecutorCores}
+
-                --driver-memory=${sparkDriverMemory}
+<!--    <action name="split_publication">-->
-                --conf spark.extraListeners=${spark2ExtraListeners}
+<!--        <spark xmlns="uri:oozie:spark-action:0.2">-->
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+<!--            <master>yarn</master>-->
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+<!--            <mode>cluster</mode>-->
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+<!--            <name>Split dumped result for community</name>-->
-                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+<!--            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>-->
-            </spark-opts>
+<!--            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
-            <arg>--sourcePath</arg><arg>${workingDir}/ext/dataset</arg>
+<!--            <spark-opts>-->
-            <arg>--outputPath</arg><arg>${outputPath}</arg>
+<!--                &#45;&#45;executor-memory=${sparkExecutorMemory}-->
-            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>
+<!--                &#45;&#45;executor-cores=${sparkExecutorCores}-->
-            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+<!--                &#45;&#45;driver-memory=${sparkDriverMemory}-->
-        </spark>
+<!--                &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
-        <ok to="join_split"/>
+<!--                &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
-        <error to="Kill"/>
+<!--                &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
-    </action>
+<!--                &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
-    <action name="split_orp">
+<!--                &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
-        <spark xmlns="uri:oozie:spark-action:0.2">
+<!--            </spark-opts>-->
-            <master>yarn</master>
+<!--            <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/ext/publication</arg>-->
-            <mode>cluster</mode>
+<!--            <arg>&#45;&#45;outputPath</arg><arg>${outputPath}</arg>-->
-            <name>Split dumped result for community</name>
+<!--            <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>-->
-            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>
+<!--            <arg>&#45;&#45;isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+<!--        </spark>-->
-            <spark-opts>
+<!--        <ok to="join_split"/>-->
-                --executor-memory=${sparkExecutorMemory}
+<!--        <error to="Kill"/>-->
-                --executor-cores=${sparkExecutorCores}
+<!--    </action>-->
-                --driver-memory=${sparkDriverMemory}
+
-                --conf spark.extraListeners=${spark2ExtraListeners}
+<!--    <action name="split_dataset">-->
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+<!--        <spark xmlns="uri:oozie:spark-action:0.2">-->
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+<!--            <master>yarn</master>-->
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+<!--            <mode>cluster</mode>-->
-                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+<!--            <name>Split dumped result for community</name>-->
-            </spark-opts>
+<!--            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>-->
-            <arg>--sourcePath</arg><arg>${workingDir}/ext/orp</arg>
+<!--            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
-            <arg>--outputPath</arg><arg>${outputPath}</arg>
+<!--            <spark-opts>-->
-            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>
+<!--                &#45;&#45;executor-memory=${sparkExecutorMemory}-->
-            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+<!--                &#45;&#45;executor-cores=${sparkExecutorCores}-->
-        </spark>
+<!--                &#45;&#45;driver-memory=${sparkDriverMemory}-->
-        <ok to="join_split"/>
+<!--                &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
-        <error to="Kill"/>
+<!--                &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
-    </action>
+<!--                &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
-    <action name="split_software">
+<!--                &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
-        <spark xmlns="uri:oozie:spark-action:0.2">
+<!--                &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
-            <master>yarn</master>
+<!--            </spark-opts>-->
-            <mode>cluster</mode>
+<!--            <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/ext/dataset</arg>-->
-            <name>Split dumped result for community</name>
+<!--            <arg>&#45;&#45;outputPath</arg><arg>${outputPath}</arg>-->
-            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>
+<!--            <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>-->
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+<!--            <arg>&#45;&#45;isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
-            <spark-opts>
+<!--        </spark>-->
-                --executor-memory=${sparkExecutorMemory}
+<!--        <ok to="join_split"/>-->
-                --executor-cores=${sparkExecutorCores}
+<!--        <error to="Kill"/>-->
-                --driver-memory=${sparkDriverMemory}
+<!--    </action>-->
-                --conf spark.extraListeners=${spark2ExtraListeners}
+<!--    <action name="split_orp">-->
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+<!--        <spark xmlns="uri:oozie:spark-action:0.2">-->
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+<!--            <master>yarn</master>-->
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+<!--            <mode>cluster</mode>-->
-                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+<!--            <name>Split dumped result for community</name>-->
-            </spark-opts>
+<!--            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>-->
-            <arg>--sourcePath</arg><arg>${workingDir}/ext/software</arg>
+<!--            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
-            <arg>--outputPath</arg><arg>${outputPath}</arg>
+<!--            <spark-opts>-->
-            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
+<!--                &#45;&#45;executor-memory=${sparkExecutorMemory}-->
-            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+<!--                &#45;&#45;executor-cores=${sparkExecutorCores}-->
-        </spark>
+<!--                &#45;&#45;driver-memory=${sparkDriverMemory}-->
-        <ok to="join_split"/>
+<!--                &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
-        <error to="Kill"/>
+<!--                &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
-    </action>
+<!--                &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
+<!--                &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
+<!--                &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
+<!--            </spark-opts>-->
+<!--            <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/ext/orp</arg>-->
+<!--            <arg>&#45;&#45;outputPath</arg><arg>${outputPath}</arg>-->
+<!--            <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>-->
+<!--            <arg>&#45;&#45;isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
+<!--        </spark>-->
+<!--        <ok to="join_split"/>-->
+<!--        <error to="Kill"/>-->
+<!--    </action>-->
+<!--    <action name="split_software">-->
+<!--        <spark xmlns="uri:oozie:spark-action:0.2">-->
+<!--            <master>yarn</master>-->
+<!--            <mode>cluster</mode>-->
+<!--            <name>Split dumped result for community</name>-->
+<!--            <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>-->
+<!--            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
+<!--            <spark-opts>-->
+<!--                &#45;&#45;executor-memory=${sparkExecutorMemory}-->
+<!--                &#45;&#45;executor-cores=${sparkExecutorCores}-->
+<!--                &#45;&#45;driver-memory=${sparkDriverMemory}-->
+<!--                &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
+<!--                &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
+<!--                &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
+<!--                &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
+<!--                &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
+<!--            </spark-opts>-->
+<!--            <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/ext/software</arg>-->
+<!--            <arg>&#45;&#45;outputPath</arg><arg>${outputPath}</arg>-->
+<!--            <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>-->
+<!--            <arg>&#45;&#45;isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
+<!--        </spark>-->
+<!--        <ok to="join_split"/>-->
+<!--        <error to="Kill"/>-->
+<!--    </action>-->
 
 <!--    <join name="join_split" to="loadInZenodo"/>-->
-    <join name="join_split" to="End"/>
+<!--    <join name="join_split" to="End"/>-->
 
 <!--    <action name="loadInZenodo">-->
 <!--        <spark xmlns="uri:oozie:spark-action:0.2">-->

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/project_input_parameters.json

@@ -19,12 +19,6 @@
 		"paramDescription": "true if the spark session is managed, false otherwise",
 		"paramRequired": false
 	},
-	{
-		"paramName":"tn",
-		"paramLongName":"resultTableName",
-		"paramDescription": "the name of the result table we are currently working on",
-		"paramRequired": true
-	},
 	{
 		"paramName": "pip",
 		"paramLongName": "preparedInfoPath",

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json

@@ -24,12 +24,6 @@
 		"paramDescription": "true if the spark session is managed, false otherwise",
 		"paramRequired": false
 	},
-	{
-		"paramName":"tn",
-		"paramLongName":"resultTableName",
-		"paramDescription": "the name of the result table we are currently working on",
-		"paramRequired": true
-	},
 	{
 		"paramName":"map",
 		"paramLongName":"communityMap",