2018-10-11 15:19:20 +02:00
|
|
|
package eu.dnetlib.reporter;
|
|
|
|
|
2018-10-18 11:56:26 +02:00
|
|
|
import eu.dnetlib.pace.util.Reporter;
|
2018-10-11 15:19:20 +02:00
|
|
|
import org.apache.commons.logging.Log;
|
|
|
|
import org.apache.commons.logging.LogFactory;
|
2019-05-21 16:38:26 +02:00
|
|
|
import org.apache.spark.util.LongAccumulator;
|
|
|
|
import scala.Serializable;
|
2018-10-11 15:19:20 +02:00
|
|
|
import scala.Tuple2;
|
|
|
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.List;
|
2019-05-21 16:38:26 +02:00
|
|
|
import java.util.Map;
|
2018-10-11 15:19:20 +02:00
|
|
|
|
2019-05-21 16:38:26 +02:00
|
|
|
public class SparkReporter implements Serializable {
|
2018-10-11 15:19:20 +02:00
|
|
|
|
|
|
|
final List<Tuple2<String, String>> report = new ArrayList<>();
|
|
|
|
private static final Log log = LogFactory.getLog(SparkReporter.class);
|
|
|
|
|
2019-05-21 16:38:26 +02:00
|
|
|
public SparkReporter(){}
|
|
|
|
|
|
|
|
public void incrementCounter(String counterGroup, String counterName, long delta, Map<String, LongAccumulator> accumulators) {
|
2018-10-11 15:19:20 +02:00
|
|
|
|
2019-05-21 16:38:26 +02:00
|
|
|
final String accumulatorName = String.format("%s::%s", counterGroup, counterName);
|
|
|
|
if (accumulators.containsKey(accumulatorName)){
|
|
|
|
accumulators.get(accumulatorName).add(delta);
|
|
|
|
}
|
2018-10-11 15:19:20 +02:00
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
public void emit(String type, String from, String to) {
|
|
|
|
report.add(new Tuple2<>(from, to));
|
|
|
|
}
|
|
|
|
|
|
|
|
public List<Tuple2<String, String>> getReport() {
|
|
|
|
return report;
|
|
|
|
}
|
|
|
|
}
|