sourcePath
the source path
writeUpdate
writes the information found for the update. No double check done if the information is already present
saveGraph
writes new version of the graph after the propagation step
Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
${jobTracker}
${nameNode}
${nameNode}/${sourcePath}/relation
${nameNode}/${workingDir}/resulttoorganization_propagation/relation
yarn
cluster
PrepareResultOrganizationAssociation
eu.dnetlib.dhp.resulttoorganizationfrominstrepo.PrepareResultInstRepoAssociation
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--sourcePath${sourcePath}
--hive_metastore_uris${hive_metastore_uris}
--datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
--alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
yarn
cluster
resultToOrganizationFromInstRepoPropagationForPublications
eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}/publication
--hive_metastore_uris${hive_metastore_uris}
--writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Publication
--outputPath${workingDir}/resulttoorganization_propagation/relation
--datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
--alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
yarn
cluster
resultToOrganizationFromInstRepoPropagationForDataset
eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}/dataset
--hive_metastore_uris${hive_metastore_uris}
--writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset
--outputPath${workingDir}/resulttoorganization_propagation/relation
--datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
--alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
yarn
cluster
resultToOrganizationFromInstRepoPropagationForORP
eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}/otherresearchproduct
--hive_metastore_uris${hive_metastore_uris}
--writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct
--outputPath${workingDir}/resulttoorganization_propagation/relation
--datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
--alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked
yarn
cluster
resultToOrganizationFromInstRepoPropagationForSoftware
eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}/software
--hive_metastore_uris${hive_metastore_uris}
--writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Software
--outputPath${workingDir}/resulttoorganization_propagation/relation
--datasourceOrganizationPath${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization
--alreadyLinkedPath${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked