sourcePath
the source path
entity
the entity that should be processed
dedupConf
the dedup Configuration
targetPath
the target path
sparkDriverMemory
memory for driver process
sparkExecutorMemory
memory for individual executor
Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
${jobTracker}
${nameNode}
yarn-cluster
cluster
Create Similarity Relations
eu.dnetlib.dedup.SparkCreateSimRels
dhp-dedup-${projectVersion}.jar
--executor-memory ${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
${sparkExtraOPT}
-mtyarn-cluster
--sourcePath${sourcePath}
--targetPath${targetPath}
--entity${entity}
--dedupConf${dedupConf}
${jobTracker}
${nameNode}
yarn-cluster
cluster
Create Connected Components
eu.dnetlib.dedup.SparkCreateConnectedComponent
dhp-dedup-${projectVersion}.jar
--executor-memory ${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
${sparkExtraOPT}
-mtyarn-cluster
--sourcePath${sourcePath}
--targetPath${targetPath}
--entity${entity}
--dedupConf${dedupConf}
${jobTracker}
${nameNode}
yarn-cluster
cluster
Create Dedup Record
eu.dnetlib.dedup.SparkCreateDedupRecord
dhp-dedup-${projectVersion}.jar
--executor-memory ${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
${sparkExtraOPT}
-mtyarn-cluster
--sourcePath${sourcePath}
--dedupPath${targetPath}
--entity${entity}
--dedupConf${dedupConf}
${jobTracker}
${nameNode}
yarn-cluster
cluster
Propagate Dedup Relations
eu.dnetlib.dedup.SparkPropagateRelationsJob
dhp-dedup-${projectVersion}.jar
--executor-memory ${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
${sparkExtraOPT}
-mtyarn-cluster
--mergeRelPath${targetPath}/${entity}/mergeRel
--relationPath${sourcePath}/relation
--targetRelPath${targetPath}/${entity}/relation_propagated
${jobTracker}
${nameNode}
yarn-cluster
cluster
Update ${entity} and add DedupRecord
eu.dnetlib.dedup.SparkUpdateEntityJob
dhp-dedup-${projectVersion}.jar
--executor-memory ${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
${sparkExtraOPT}
-mtyarn-cluster
--entityPath${sourcePath}/${entity}
--mergeRelPath${targetPath}/${entity}/mergeRel
--entity${entity}
--dedupRecordPath${targetPath}/${entity}/dedup_records
--targetPath${targetPath}/${entity}/updated_record
${jobTracker}
${nameNode}
yarn-cluster
cluster
Update ${entity} set deleted by Inference
eu.dnetlib.dedup.SparkUpdateEntityJob
dhp-dedup-${projectVersion}.jar
--executor-memory ${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory}
${sparkExtraOPT}
-mtyarn-cluster
--entityPath${targetPath}/${entity}/relation_propagated
--mergeRelPath${targetPath}/${entity}/mergeRel
--entityrelation
--dedupRecordPath${targetPath}/${entity}/dedup_records
--targetPath${targetPath}/${entity}/updated_relation