hive_db_name
the target hive database name
sparkDriverMemory
memory for driver process
sparkExecutorMemory
memory for individual executor
sparkExecutorCores
number of cores used by single executor
spark2YarnHistoryServerAddress
spark 2.* yarn history server address
spark2EventLogDir
spark 2.* event log dir location
${jobTracker}
${nameNode}
mapreduce.job.queuename
${queueName}
oozie.launcher.mapred.job.queue.name
${oozieLauncherQueueName}
${wf:conf('reuseRecords') eq false}
${wf:conf('reuseRecords') eq true}
Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
yarn
cluster
build_adjacency_lists
eu.dnetlib.dhp.oa.provision.SparkXmlRecordBuilderJob_v2
dhp-graph-provision-${projectVersion}.jar
--executor-cores ${sparkExecutorCoresForJoining}
--executor-memory ${sparkExecutorMemoryForJoining}
--driver-memory=${sparkDriverMemoryForJoining}
--conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener"
--conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener"
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.network.timeout=10000000
-mt yarn
-is ${isLookupUrl}
-t ${otherDsTypeId}
--sourcePath${sourcePath}
--outputPath${outputPath}
yarn
cluster
to_solr_index
eu.dnetlib.dhp.oa.provision.SparkXmlIndexingJob
dhp-graph-provision-${projectVersion}.jar
--executor-cores ${sparkExecutorCoresForIndexing}
--executor-memory ${sparkExecutorMemoryForIndexing}
--driver-memory=${sparkDriverMemoryForIndexing}
--conf spark.dynamicAllocation.maxExecutors=${sparkExecutorCoresForIndexing}
--conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener"
--conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener"
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-mt yarn
-is ${isLookupUrl}
--sourcePath${outputPath}/xml
--format${format}
--batchSize${batchSize}