workingPath
the base path to store hdfs file
graphRawPath
the graph Raw base path
postgresURL
the postgres URL to access to the database
postgresUser
the user postgres
postgresPassword
the password postgres
mongourl
mongoDB url, example: mongodb://[username:password@]host[:port]
mongoDb
mongo database
sparkDriverMemory
memory for driver process
sparkExecutorMemory
memory for individual executor
sparkExecutorCores
number of cores used by single executor
Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]
${jobTracker}
${nameNode}
eu.dnetlib.dhp.migration.MigrateDbEntitiesApplication
-p${workingPath}/db_entities
-n${nameNode}
-u${hdfsUser}
-dburl${postgresURL}
-dbuser${postgresUser}
-dbpasswd${postgresPassword}
${jobTracker}
${nameNode}
eu.dnetlib.dhp.migration.MigrateDbEntitiesApplication
-p${workingPath}/db_claims
-n${nameNode}
-u${hdfsUser}
-dburl${postgresURL}
-dbuser${postgresUser}
-dbpasswd${postgresPassword}
-aclaims
${jobTracker}
${nameNode}
eu.dnetlib.dhp.migration.MigrateMongoMdstoresApplication
-p${workingPath}/odf_entities
-n${nameNode}
-u${hdfsUser}
-mongourl${mongourl}
-db${mongoDb}
-fODF
-lstore
-icleaned
-pgurl${postgresURL}
-pguser${postgresUser}
-pgpasswd${postgresPassword}
${jobTracker}
${nameNode}
eu.dnetlib.dhp.migration.MigrateMongoMdstoresApplication
-p${workingPath}/oaf_entities
-n${nameNode}
-u${hdfsUser}
-mongourl${mongourl}
-db${mongoDb}
-fOAF
-lstore
-icleaned
-pgurl${postgresURL}
-pguser${postgresUser}
-pgpasswd${postgresPassword}
${jobTracker}
${nameNode}
eu.dnetlib.dhp.migration.MigrateMongoMdstoresApplication
-p${workingPath}/odf_claims
-n${nameNode}
-u${hdfsUser}
-mongourl${mongourl}
-db${mongoDb}
-fODF
-lstore
-iclaim
-pgurl${postgresURL}
-pguser${postgresUser}
-pgpasswd${postgresPassword}
${jobTracker}
${nameNode}
eu.dnetlib.dhp.migration.MigrateMongoMdstoresApplication
-p${workingPath}/oaf_claims
-n${nameNode}
-u${hdfsUser}
-mongourl${mongourl}
-db${mongoDb}
-fOAF
-lstore
-iclaim
-pgurl${postgresURL}
-pguser${postgresUser}
-pgpasswd${postgresPassword}
${jobTracker}
${nameNode}
yarn-cluster
cluster
ExtractEntities
eu.dnetlib.dhp.migration.ExtractEntitiesFromHDFSJob
dhp-aggregation-${projectVersion}.jar
--executor-memory ${sparkExecutorMemory} --executor-cores ${sparkExecutorCores} --driver-memory=${sparkDriverMemory} --conf spark.extraListeners="com.cloudera.spark.lineage.NavigatorAppListener" --conf spark.sql.queryExecutionListeners="com.cloudera.spark.lineage.NavigatorQueryListener" --conf spark.sql.warehouse.dir="/user/hive/warehouse"
-mt yarn-cluster
-s${workingPath}/db_entities,${workingPath}/oaf_entities,${workingPath}/odf_entities
-g${graphRawPath}