Improved parallelization on transformation wf on hadoop

This commit is contained in:
Sandro La Bruzzo 2021-04-19 15:17:25 +02:00
parent cdfe01bbae
commit 524e5f3092
1 changed files with 10 additions and 0 deletions

View File

@ -149,4 +149,14 @@ public class TransformSparkJobNode {
}
}
/**
* Calculates the number of partitions allocating at most @rpt records for a single transformation task.
* @param totalInput
* @param rpt
* @return
*/
private static int getRepartitionNumber(long totalInput, Integer rpt) {
return (int) (totalInput / rpt);
}
}