|
|
|
@ -505,21 +505,21 @@
|
|
|
|
|
<arg>--maxEventsForTopic</arg><arg>${maxIndexedEventsForDsAndTopic}</arg>
|
|
|
|
|
<arg>--brokerApiBaseUrl</arg><arg>${brokerApiBaseUrl}</arg>
|
|
|
|
|
</spark>
|
|
|
|
|
<ok to="index_notifications"/>
|
|
|
|
|
<ok to="stats"/>
|
|
|
|
|
<error to="Kill"/>
|
|
|
|
|
</action>
|
|
|
|
|
|
|
|
|
|
<action name="index_notifications">
|
|
|
|
|
<action name="stats">
|
|
|
|
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
|
|
|
<master>yarn</master>
|
|
|
|
|
<mode>cluster</mode>
|
|
|
|
|
<name>IndexNotificationsOnESJob</name>
|
|
|
|
|
<class>eu.dnetlib.dhp.broker.oa.IndexNotificationsJob</class>
|
|
|
|
|
<name>GenerateStatsJob</name>
|
|
|
|
|
<class>eu.dnetlib.dhp.broker.oa.GenerateStatsJob</class>
|
|
|
|
|
<jar>dhp-broker-events-${projectVersion}.jar</jar>
|
|
|
|
|
<spark-opts>
|
|
|
|
|
--executor-cores=${sparkExecutorCores}
|
|
|
|
|
--executor-memory=${sparkExecutorMemory}
|
|
|
|
|
--driver-memory=${sparkDriverMemory}
|
|
|
|
|
--conf spark.dynamicAllocation.maxExecutors="8"
|
|
|
|
|
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
|
|
|
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
|
|
|
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
|
|
@ -527,29 +527,26 @@
|
|
|
|
|
--conf spark.sql.shuffle.partitions=3840
|
|
|
|
|
</spark-opts>
|
|
|
|
|
<arg>--outputDir</arg><arg>${outputDir}</arg>
|
|
|
|
|
<arg>--index</arg><arg>${esNotificationsIndexName}</arg>
|
|
|
|
|
<arg>--esHost</arg><arg>${esIndexHost}</arg>
|
|
|
|
|
<arg>--esBatchWriteRetryCount</arg><arg>${esBatchWriteRetryCount}</arg>
|
|
|
|
|
<arg>--esBatchWriteRetryWait</arg><arg>${esBatchWriteRetryWait}</arg>
|
|
|
|
|
<arg>--esBatchSizeEntries</arg><arg>${esBatchSizeEntries}</arg>
|
|
|
|
|
<arg>--esNodesWanOnly</arg><arg>${esNodesWanOnly}</arg>
|
|
|
|
|
<arg>--dbUrl</arg><arg>${brokerDbUrl}</arg>
|
|
|
|
|
<arg>--dbUser</arg><arg>${brokerDbUser}</arg>
|
|
|
|
|
<arg>--dbPassword</arg><arg>${brokerDbPassword}</arg>
|
|
|
|
|
<arg>--brokerApiBaseUrl</arg><arg>${brokerApiBaseUrl}</arg>
|
|
|
|
|
</spark>
|
|
|
|
|
<ok to="stats"/>
|
|
|
|
|
<ok to="index_notifications"/>
|
|
|
|
|
<error to="Kill"/>
|
|
|
|
|
</action>
|
|
|
|
|
|
|
|
|
|
<action name="stats">
|
|
|
|
|
<action name="index_notifications">
|
|
|
|
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
|
|
|
<master>yarn</master>
|
|
|
|
|
<mode>cluster</mode>
|
|
|
|
|
<name>GenerateStatsJob</name>
|
|
|
|
|
<class>eu.dnetlib.dhp.broker.oa.GenerateStatsJob</class>
|
|
|
|
|
<name>IndexNotificationsOnESJob</name>
|
|
|
|
|
<class>eu.dnetlib.dhp.broker.oa.IndexNotificationsJob</class>
|
|
|
|
|
<jar>dhp-broker-events-${projectVersion}.jar</jar>
|
|
|
|
|
<spark-opts>
|
|
|
|
|
--executor-cores=${sparkExecutorCores}
|
|
|
|
|
--executor-memory=${sparkExecutorMemory}
|
|
|
|
|
--driver-memory=${sparkDriverMemory}
|
|
|
|
|
--conf spark.dynamicAllocation.maxExecutors="8"
|
|
|
|
|
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
|
|
|
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
|
|
|
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
|
|
@ -557,15 +554,18 @@
|
|
|
|
|
--conf spark.sql.shuffle.partitions=3840
|
|
|
|
|
</spark-opts>
|
|
|
|
|
<arg>--outputDir</arg><arg>${outputDir}</arg>
|
|
|
|
|
<arg>--dbUrl</arg><arg>${brokerDbUrl}</arg>
|
|
|
|
|
<arg>--dbUser</arg><arg>${brokerDbUser}</arg>
|
|
|
|
|
<arg>--dbPassword</arg><arg>${brokerDbPassword}</arg>
|
|
|
|
|
<arg>--index</arg><arg>${esNotificationsIndexName}</arg>
|
|
|
|
|
<arg>--esHost</arg><arg>${esIndexHost}</arg>
|
|
|
|
|
<arg>--esBatchWriteRetryCount</arg><arg>${esBatchWriteRetryCount}</arg>
|
|
|
|
|
<arg>--esBatchWriteRetryWait</arg><arg>${esBatchWriteRetryWait}</arg>
|
|
|
|
|
<arg>--esBatchSizeEntries</arg><arg>${esBatchSizeEntries}</arg>
|
|
|
|
|
<arg>--esNodesWanOnly</arg><arg>${esNodesWanOnly}</arg>
|
|
|
|
|
<arg>--brokerApiBaseUrl</arg><arg>${brokerApiBaseUrl}</arg>
|
|
|
|
|
</spark>
|
|
|
|
|
<ok to="End"/>
|
|
|
|
|
<error to="Kill"/>
|
|
|
|
|
</action>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<end name="End"/>
|
|
|
|
|
|
|
|
|
|
</workflow-app>
|