sourcePath
the source path
allowedsemrels
the semantic relationships allowed for propagation
writeUpdate
writes the information found for the update. No double check done if the information is already present
saveGraph
writes new version of the graph after the propagation step
Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
${jobTracker}
${nameNode}
${nameNode}/${sourcePath}/relation
${nameNode}/${workingDir}/orcid_propagation/relation
yarn
cluster
ORCIDPropagation-PreparePhase1-Publications
eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}
--hive_metastore_uris${hive_metastore_uris}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Publication
--outputPath${workingDir}/preparedInfo/targetOrcidAssoc
--allowedsemrels${allowedsemrels}
yarn
cluster
ORCIDPropagation-PreparePhase1-Dataset
eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}
--hive_metastore_uris${hive_metastore_uris}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset
--outputPath${workingDir}/preparedInfo/targetOrcidAssoc
--allowedsemrels${allowedsemrels}
yarn
cluster
ORCIDPropagation-PreparePhase1-ORP
eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}
--hive_metastore_uris${hive_metastore_uris}
--resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct
--outputPath${workingDir}/preparedInfo/targetOrcidAssoc
--allowedsemrels${allowedsemrels}
yarn
cluster
ORCIDPropagation-PreparePhase1-Software
eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}
--hive_metastore_uris${hive_metastore_uris}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Software
--outputPath${workingDir}/preparedInfo/targetOrcidAssoc
--allowedsemrels${allowedsemrels}
yarn
cluster
ORCIDPropagation-PreparePhase2
eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${workingDir}/preparedInfo/targetOrcidAssoc
--outputPath${workingDir}/preparedInfo/mergedOrcidAssoc
yarn
cluster
ORCIDPropagation-Publication
eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--possibleUpdatesPath${workingDir}/preparedInfo/mergedOrcidAssoc
--sourcePath${sourcePath}/publication
--hive_metastore_uris${hive_metastore_uris}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Publication
--outputPath${workingDir}/orcid_propagation/publication
yarn
cluster
ORCIDPropagation-Dataset
eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--possibleUpdatesPath${workingDir}/preparedInfo/mergedOrcidAssoc
--sourcePath${sourcePath}/dataset
--hive_metastore_uris${hive_metastore_uris}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset
--outputPath${workingDir}/orcid_propagation/dataset
yarn
cluster
ORCIDPropagation-ORP
eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--possibleUpdatesPath${workingDir}/preparedInfo/mergedOrcidAssoc
--sourcePath${sourcePath}/otherresearchproduct
--hive_metastore_uris${hive_metastore_uris}
--resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct
--outputPath${workingDir}/orcid_propagation/otherresearchproduct
yarn
cluster
ORCIDPropagation-Software
eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--possibleUpdatesPath${workingDir}/preparedInfo/mergedOrcidAssoc
--sourcePath${sourcePath}/software
--hive_metastore_uris${hive_metastore_uris}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Software
--outputPath${workingDir}/orcid_propagation/software