forked from D-Net/dnet-hadoop
Improved parallelization on transformation wf on hadoop
This commit is contained in:
parent
cdfe01bbae
commit
524e5f3092
|
@ -149,4 +149,14 @@ public class TransformSparkJobNode {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates the number of partitions allocating at most @rpt records for a single transformation task.
|
||||||
|
* @param totalInput
|
||||||
|
* @param rpt
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
private static int getRepartitionNumber(long totalInput, Integer rpt) {
|
||||||
|
return (int) (totalInput / rpt);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue