forked from D-Net/dnet-hadoop
Improved parallelization on transformation wf on hadoop
This commit is contained in:
parent
cdfe01bbae
commit
524e5f3092
|
@ -149,4 +149,14 @@ public class TransformSparkJobNode {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the number of partitions allocating at most @rpt records for a single transformation task.
|
||||
* @param totalInput
|
||||
* @param rpt
|
||||
* @return
|
||||
*/
|
||||
private static int getRepartitionNumber(long totalInput, Integer rpt) {
|
||||
return (int) (totalInput / rpt);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue