sourcePath
the source path
whitelist
the white list
allowedtypes
the allowed types
sparkDriverMemory
memory for driver process
sparkExecutorMemory
memory for individual executor
sparkExecutorCores
number of cores used by single executor
sparkExecutorNumber
number of executors used
writeUpdate
writes the information found for the update. No double check done if the information is already present
saveGraph
writes new version of the graph after the propagation step
Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
yarn
cluster
PrepareDatasourceCountryAssociation
eu.dnetlib.dhp.countrypropagation.PrepareDatasourceCountryAssociation
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--sourcePath${sourcePath}
--whitelist${whitelist}
--allowedtypes${allowedtypes}
--hive_metastore_uris${hive_metastore_uris}
--outputPath${workingDir}/country_propagation/preparedInfo
yarn
cluster
countryPropagationForPublications
eu.dnetlib.dhp.countrypropagation.SparkCountryPropagationJob2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}/publication
--hive_metastore_uris${hive_metastore_uris}
--writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Publication
--outputPath${workingDir}/country_propagation/publication
--preparedInfoPath${workingDir}/country_propagation/preparedInfo
yarn
cluster
countryPropagationForDataset
eu.dnetlib.dhp.countrypropagation.SparkCountryPropagationJob2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}/dataset
--hive_metastore_uris${hive_metastore_uris}
--writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset
--outputPath${workingDir}/country_propagation/dataset
--preparedInfoPath${workingDir}/country_propagation/preparedInfo
yarn
cluster
countryPropagationForORP
eu.dnetlib.dhp.countrypropagation.SparkCountryPropagationJob2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}/otherresearchproduct
--hive_metastore_uris${hive_metastore_uris}
--writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct
--outputPath${workingDir}/country_propagation/otherresearchproduct
--preparedInfoPath${workingDir}/country_propagation/preparedInfo
yarn
cluster
countryPropagationForSoftware
eu.dnetlib.dhp.countrypropagation.SparkCountryPropagationJob2
dhp-propagation-${projectVersion}.jar
--executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.dynamicAllocation.enabled=true
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
--sourcePath${sourcePath}/software
--hive_metastore_uris${hive_metastore_uris}
--writeUpdate${writeUpdate}
--saveGraph${saveGraph}
--resultTableNameeu.dnetlib.dhp.schema.oaf.Software
--outputPath${workingDir}/country_propagation/software
--preparedInfoPath${workingDir}/country_propagation/preparedInfo