D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

increased memory

This commit is contained in:
Michele Artini 2024-02-14 14:40:27 +01:00
parent 2e11197142
commit 773346f638
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/base/BaseAnalyzerJob.java
View File

@ -114,6 +114,10 @@ public class BaseAnalyzerJob {
final long i = recordsCounter.incrementAndGet(); final long i = recordsCounter.incrementAndGet();
if ((i % 10000) == 0) { if ((i % 10000) == 0) {
log.info("# Read records: " + i); log.info("# Read records: " + i);
log.info("# fields: " + fields.size());
log.info("# types: " + types.size());
log.info("# collections: " + collections.size());
log.info("# totals: " + totals.size());
} }
final List<String> recTypes = new ArrayList<>(); final List<String> recTypes = new ArrayList<>();

2
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/plugin/base/oozie_app/config-default.xml
View File

@ -49,7 +49,7 @@
</property> </property>
<property> <property>
<name>sparkExecutorMemory</name> <name>sparkExecutorMemory</name>
<value>6G</value> <value>10G</value>
</property> </property>
<property> <property>
<name>sparkExecutorCores</name> <name>sparkExecutorCores</name>