diff --git a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_propagationresulaffiliationfrominstrepo_parameters.json b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_propagationresulaffiliationfrominstrepo_parameters.json
index 9da25874a..d2b076c82 100644
--- a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_propagationresulaffiliationfrominstrepo_parameters.json
+++ b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_propagationresulaffiliationfrominstrepo_parameters.json
@@ -11,17 +11,11 @@
"paramDescription": "the hive metastore uris",
"paramRequired": true
},
- {
- "paramName":"wu",
- "paramLongName":"writeUpdate",
- "paramDescription": "true if the update must be writte. No double check if information is already present",
- "paramRequired": true
- },
{
"paramName":"sg",
"paramLongName":"saveGraph",
"paramDescription": "true if the new version of the graph must be saved",
- "paramRequired": true
+ "paramRequired": false
},
{
"paramName":"dop",
diff --git a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml
index 245c4f556..f1495e03b 100644
--- a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml
@@ -5,12 +5,8 @@
the source path
- writeUpdate
- writes the information found for the update. No double check done if the information is already present
-
-
- saveGraph
- writes new version of the graph after the propagation step
+ outputPath
+ sets the outputPath
@@ -22,18 +18,29 @@
-
+
+
+
+
+
+
+
+
+
-
+
-
+
+
+
+
@@ -41,18 +48,17 @@
${jobTracker}
${nameNode}
${nameNode}/${sourcePath}/relation
- ${nameNode}/${workingDir}/resulttoorganization_propagation/relation
+ ${nameNode}/${outputPath}/relation
-
${jobTracker}
${nameNode}
${nameNode}/${sourcePath}/publication
- ${nameNode}/${workingDir}/resulttoorganization_propagation/publication
+ ${nameNode}/${outputPath}/publication
@@ -63,7 +69,7 @@
${jobTracker}
${nameNode}
${nameNode}/${sourcePath}/dataset
- ${nameNode}/${workingDir}/resulttoorganization_propagation/dataset
+ ${nameNode}/${outputPath}/dataset
@@ -74,7 +80,7 @@
${jobTracker}
${nameNode}
${nameNode}/${sourcePath}/otherresearchproduct
- ${nameNode}/${workingDir}/resulttoorganization_propagation/otherresearchproduct
+ ${nameNode}/${outputPath}/otherresearchproduct
@@ -84,11 +90,44 @@
${jobTracker}
${nameNode}
${nameNode}/${sourcePath}/software
- ${nameNode}/${workingDir}/resulttoorganization_propagation/software
+ ${nameNode}/${outputPath}/software
+
+
+ ${jobTracker}
+ ${nameNode}
+ ${nameNode}/${sourcePath}/organization
+ ${nameNode}/${outputPath}/organization
+
+
+
+
+
+
+
+ ${jobTracker}
+ ${nameNode}
+ ${nameNode}/${sourcePath}/project
+ ${nameNode}/${outputPath}/project
+
+
+
+
+
+
+
+ ${jobTracker}
+ ${nameNode}
+ ${nameNode}/${sourcePath}/datasource
+ ${nameNode}/${outputPath}/datasource
+
+
+
+
+
@@ -108,11 +147,11 @@
--sourcePath${sourcePath}
--hive_metastore_uris${hive_metastore_uris}
- --datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
- --alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
+ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization
+ --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked
-
+
@@ -142,12 +181,11 @@
--sourcePath${sourcePath}/publication
--hive_metastore_uris${hive_metastore_uris}
- --writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Publication
- --outputPath${workingDir}/resulttoorganization_propagation/relation
- --datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
- --alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
+ --outputPath${outputPath}/relation
+ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization
+ --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked
@@ -173,17 +211,15 @@
--sourcePath${sourcePath}/dataset
--hive_metastore_uris${hive_metastore_uris}
- --writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset
- --outputPath${workingDir}/resulttoorganization_propagation/relation
- --datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
- --alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
+ --outputPath${outputPath}/relation
+ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization
+ --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked
-
yarn
@@ -204,12 +240,11 @@
--sourcePath${sourcePath}/otherresearchproduct
--hive_metastore_uris${hive_metastore_uris}
- --writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct
- --outputPath${workingDir}/resulttoorganization_propagation/relation
- --datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
- --alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
+ --outputPath${outputPath}/relation
+ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization
+ --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked
@@ -235,12 +270,30 @@
--sourcePath${sourcePath}/software
--hive_metastore_uris${hive_metastore_uris}
- --writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Software
- --outputPath${workingDir}/resulttoorganization_propagation/relation
- --datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
- --alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
+ --outputPath${outputPath}/relation
+ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization
+ --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked
+ dhp-propagation-${projectVersion}.jar
+
+ --executor-cores=${sparkExecutorCores}
+ --executor-memory=${sparkExecutorMemory}
+ --driver-memory=${sparkDriverMemory}
+ --conf spark.extraListeners=${spark2ExtraListeners}
+ --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+ --conf spark.dynamicAllocation.enabled=true
+ --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
+
+ --sourcePath${sourcePath}/software
+ --hive_metastore_uris${hive_metastore_uris}
+ --saveGraph${saveGraph}
+ --resultTableNameeu.dnetlib.dhp.schema.oaf.Software
+ --outputPath${outputPath}/relation
+ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization
+ --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked