diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java index 829921dd8..09f3ffd63 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java @@ -45,6 +45,8 @@ public class MDStoreActionNode { "/eu/dnetlib/dhp/collection/mdstore_action_parameters.json"))); argumentParser.parseArgument(args); + log.info("Java Xmx: {}m", Runtime.getRuntime().maxMemory() / (1024 * 1024)); + final MDAction action = MDAction.valueOf(argumentParser.get("action")); log.info("Current action is {}", action); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/worker/CollectorWorkerApplication.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/worker/CollectorWorkerApplication.java index 6e4237bee..17f09ee5a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/worker/CollectorWorkerApplication.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/worker/CollectorWorkerApplication.java @@ -41,11 +41,13 @@ public class CollectorWorkerApplication { final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser( IOUtils .toString( - CollectorWorker.class + CollectorWorkerApplication.class .getResourceAsStream( "/eu/dnetlib/dhp/collection/collector_worker_input_parameter.json"))); argumentParser.parseArgument(args); + log.info("Java Xmx: {}m", Runtime.getRuntime().maxMemory() / (1024 * 1024)); + final String hdfsuri = argumentParser.get("namenode"); log.info("hdfsURI is {}", hdfsuri); diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/workflow.xml index fe8eea370..5497b2c50 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/oozie_app/workflow.xml @@ -41,6 +41,14 @@ collectionMode Should be REFRESH or INCREMENTAL + + + collection_java_xmx + -Xmx200m + Used to configure the heap size for the map JVM process. Should be 80% of mapreduce.map.memory.mb. + + + @@ -65,6 +73,7 @@ eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode + ${collection_java_xmx} --actionREAD_LOCK --mdStoreID${mdStoreID} --mdStoreManagerURI${mdStoreManagerURI} @@ -77,6 +86,7 @@ eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode + ${collection_java_xmx} --actionNEW_VERSION --mdStoreID${mdStoreID} --mdStoreManagerURI${mdStoreManagerURI} @@ -89,6 +99,7 @@ eu.dnetlib.dhp.collection.worker.CollectorWorkerApplication + ${collection_java_xmx} --apidescriptor${apiDescription} --namenode${nameNode} --workflowId${workflowId} @@ -108,6 +119,7 @@ eu.dnetlib.dhp.collection.worker.CollectorWorkerReporter + ${collection_java_xmx} --mdStoreVersion${wf:actionData('StartTransaction')['mdStoreVersion']} --namenode${nameNode} @@ -153,6 +165,7 @@ eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode + ${collection_java_xmx} --actionREAD_UNLOCK --mdStoreManagerURI${mdStoreManagerURI} --readMDStoreId${wf:actionData('BeginRead')['mdStoreReadLockVersion']} @@ -164,6 +177,7 @@ eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode + ${collection_java_xmx} --actionCOMMIT --namenode${nameNode} --mdStoreVersion${wf:actionData('StartTransaction')['mdStoreVersion']} @@ -184,6 +198,7 @@ eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode + ${collection_java_xmx} --actionREAD_UNLOCK --mdStoreManagerURI${mdStoreManagerURI} --readMDStoreId${wf:actionData('BeginRead')['mdStoreReadLockVersion']} @@ -195,6 +210,7 @@ eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode + ${collection_java_xmx} --actionROLLBACK --mdStoreVersion${wf:actionData('StartTransaction')['mdStoreVersion']} --mdStoreManagerURI${mdStoreManagerURI}