From bae029f8288e20c5888e1ef39489d88f2a08ce96 Mon Sep 17 00:00:00 2001
From: Claudio Atzori <claudio.atzori@isti.cnr.it>
Date: Mon, 8 Feb 2021 18:07:23 +0100
Subject: [PATCH] collection_java_xmx allows to declare the heap size allocated
 for the java actions involved in the metadata collectionw workflow

---
 .../aggregation/mdstore/MDStoreActionNode.java   |  2 ++
 .../worker/CollectorWorkerApplication.java       |  4 +++-
 .../dhp/collection/oozie_app/workflow.xml        | 16 ++++++++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java
index 829921dd8..09f3ffd63 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java
@@ -45,6 +45,8 @@ public class MDStoreActionNode {
 							"/eu/dnetlib/dhp/collection/mdstore_action_parameters.json")));
 		argumentParser.parseArgument(args);
 
+		log.info("Java Xmx: {}m", Runtime.getRuntime().maxMemory() / (1024 * 1024));
+
 		final MDAction action = MDAction.valueOf(argumentParser.get("action"));
 		log.info("Current action is {}", action);
 
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/worker/CollectorWorkerApplication.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/worker/CollectorWorkerApplication.java
index 6e4237bee..17f09ee5a 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/worker/CollectorWorkerApplication.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/worker/CollectorWorkerApplication.java
@@ -41,11 +41,13 @@ public class CollectorWorkerApplication {
 		final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
-					CollectorWorker.class
+					CollectorWorkerApplication.class
 						.getResourceAsStream(
 							"/eu/dnetlib/dhp/collection/collector_worker_input_parameter.json")));
 		argumentParser.parseArgument(args);
 
+		log.info("Java Xmx: {}m", Runtime.getRuntime().maxMemory() / (1024 * 1024));
+
 		final String hdfsuri = argumentParser.get("namenode");
 		log.info("hdfsURI is {}", hdfsuri);
 
diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/workflow.xml
index fe8eea370..5497b2c50 100644
--- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/workflow.xml
@@ -41,6 +41,14 @@
             <name>collectionMode</name>
             <description>Should be REFRESH or INCREMENTAL</description>
         </property>
+
+        <property>
+            <name>collection_java_xmx</name>
+            <value>-Xmx200m</value>
+            <description>Used to configure the heap size for the map JVM process. Should be 80% of mapreduce.map.memory.mb.</description>
+        </property>
+
+
     </parameters>
 
     <global>
@@ -65,6 +73,7 @@
     <action name="BeginRead">
         <java>
             <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <java-opts>${collection_java_xmx}</java-opts>
             <arg>--action</arg><arg>READ_LOCK</arg>
             <arg>--mdStoreID</arg><arg>${mdStoreID}</arg>
             <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
@@ -77,6 +86,7 @@
     <action name="StartTransaction">
         <java>
             <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <java-opts>${collection_java_xmx}</java-opts>
             <arg>--action</arg><arg>NEW_VERSION</arg>
             <arg>--mdStoreID</arg><arg>${mdStoreID}</arg>
             <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
@@ -89,6 +99,7 @@
     <action name="CollectionWorker">
         <java>
             <main-class>eu.dnetlib.dhp.collection.worker.CollectorWorkerApplication</main-class>
+            <java-opts>${collection_java_xmx}</java-opts>
             <arg>--apidescriptor</arg><arg>${apiDescription}</arg>
             <arg>--namenode</arg><arg>${nameNode}</arg>
             <arg>--workflowId</arg><arg>${workflowId}</arg>
@@ -108,6 +119,7 @@
     <action name="CollectorReport">
         <java>
             <main-class>eu.dnetlib.dhp.collection.worker.CollectorWorkerReporter</main-class>
+            <java-opts>${collection_java_xmx}</java-opts>
             <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
             <arg>--namenode</arg><arg>${nameNode}</arg>
         </java>
@@ -153,6 +165,7 @@
     <action name="EndRead">
         <java>
             <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <java-opts>${collection_java_xmx}</java-opts>
             <arg>--action</arg><arg>READ_UNLOCK</arg>
             <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
             <arg>--readMDStoreId</arg><arg>${wf:actionData('BeginRead')['mdStoreReadLockVersion']}</arg>
@@ -164,6 +177,7 @@
     <action name="CommitVersion">
         <java>
             <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <java-opts>${collection_java_xmx}</java-opts>
             <arg>--action</arg><arg>COMMIT</arg>
             <arg>--namenode</arg><arg>${nameNode}</arg>
             <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
@@ -184,6 +198,7 @@
     <action name="EndReadRollBack">
         <java>
             <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <java-opts>${collection_java_xmx}</java-opts>
             <arg>--action</arg><arg>READ_UNLOCK</arg>
             <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
             <arg>--readMDStoreId</arg><arg>${wf:actionData('BeginRead')['mdStoreReadLockVersion']}</arg>
@@ -195,6 +210,7 @@
     <action name="RollBack">
         <java>
             <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <java-opts>${collection_java_xmx}</java-opts>
             <arg>--action</arg><arg>ROLLBACK</arg>
             <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
             <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>