From d12028bd9e0d0824e207ad0a366921ee79be41ec Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Tue, 31 Dec 2024 10:32:40 +0100 Subject: [PATCH] [master ] removed resource package --- .../serafeim/oozie_app/config-default.xml | 30 ------ .../dump/serafeim/oozie_app/workflow.xml | 102 ------------------ 2 files changed, 132 deletions(-) delete mode 100644 dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/serafeim/oozie_app/config-default.xml delete mode 100644 dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/serafeim/oozie_app/workflow.xml diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/serafeim/oozie_app/config-default.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/serafeim/oozie_app/config-default.xml deleted file mode 100644 index d262cb6..0000000 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/serafeim/oozie_app/config-default.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - jobTracker - yarnRM - - - nameNode - hdfs://nameservice1 - - - oozie.use.system.libpath - true - - - hiveMetastoreUris - thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 - - - hiveJdbcUrl - jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000 - - - hiveDbName - openaire - - - oozie.launcher.mapreduce.user.classpath.first - true - - diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/serafeim/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/serafeim/oozie_app/workflow.xml deleted file mode 100644 index dc9ead6..0000000 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/serafeim/oozie_app/workflow.xml +++ /dev/null @@ -1,102 +0,0 @@ - - - - sourcePath - the source path - - - outputPath - the output path - - - communities - the communities whose products should be dumped - - - sparkDriverMemory - memory for driver process - - - sparkExecutorMemory - memory for individual executor - - - sparkExecutorCores - number of cores used by single executor - - - oozieActionShareLibForSpark2 - oozie action sharelib for spark 2.* - - - spark2ExtraListeners - com.cloudera.spark.lineage.NavigatorAppListener - spark 2.* extra listeners classname - - - spark2SqlQueryExecutionListeners - com.cloudera.spark.lineage.NavigatorQueryListener - spark 2.* sql query execution listeners classname - - - spark2YarnHistoryServerAddress - spark 2.* yarn history server address - - - spark2EventLogDir - spark 2.* event log dir location - - - - ${jobTracker} - ${nameNode} - - - mapreduce.job.queuename - ${queueName} - - - oozie.launcher.mapred.job.queue.name - ${oozieLauncherQueueName} - - - oozie.action.sharelib.for.spark - ${oozieActionShareLibForSpark2} - - - - - - - - yarn - cluster - select results ids connected to communities and dump relation - eu.dnetlib.dhp.oa.graph.dump.serafeim.SparkSelectResultsAndDumpRelations - dump-${projectVersion}.jar - - --executor-memory=10G - --executor-cores=3 - --driver-memory=10G - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - --conf spark.sql.shuffle.partitions=3840 - - --sourcePath${sourcePath} - --workingPath${workingDir} - --outputPath${outputPath} - --communities${communities} - - - - - - - Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - - - - \ No newline at end of file