entity
the entity that should be processed
entityPath
the source path
mergeRelPath
the target path
dedupRecordPath
the target path
master
the target path
sparkDriverMemory
memory for driver process
sparkExecutorMemory
memory for individual executor
Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
${jobTracker}
${nameNode}
${master}
cluster
Update ${entity} and add DedupRecord
eu.dnetlib.dedup.SparkUpdateEntityJob
dhp-dedup-${projectVersion}.jar
--executor-memory ${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
--num-executors 100
--conf spark.yarn.jars="hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2"
-mt${master}
--entityPath${entityPath}
--mergeRelPath${mergeRelPath}
--entity${entity}
--dedupRecordPath${dedupRecordPath}