70 lines
2.5 KiB
Java
70 lines
2.5 KiB
Java
|
|
package eu.dnetlib.dhp.collection.worker;
|
|
|
|
import static eu.dnetlib.dhp.aggregation.common.AggregationConstants.REPORT_FILE_NAME;
|
|
import static eu.dnetlib.dhp.aggregation.common.AggregationUtility.*;
|
|
import static eu.dnetlib.dhp.aggregation.common.AggregationUtility.MAPPER;
|
|
import static eu.dnetlib.dhp.application.ApplicationUtils.getHadoopConfiguration;
|
|
import static eu.dnetlib.dhp.application.ApplicationUtils.populateOOZIEEnv;
|
|
|
|
import java.io.IOException;
|
|
import java.util.Objects;
|
|
|
|
import org.apache.commons.cli.ParseException;
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion;
|
|
import eu.dnetlib.dhp.aggregation.common.AggregationUtility;
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
import eu.dnetlib.dhp.collection.worker.utils.CollectorPluginReport;
|
|
import eu.dnetlib.dhp.collection.worker.utils.UnknownCollectorPluginException;
|
|
|
|
/**
|
|
* CollectorWorkerReporter
|
|
*/
|
|
public class CollectorWorkerReporter {
|
|
|
|
private static final Logger log = LoggerFactory.getLogger(CollectorWorkerReporter.class);
|
|
|
|
/**
|
|
* @param args
|
|
*/
|
|
public static void main(final String[] args) throws IOException, ParseException, CollectorException {
|
|
|
|
final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser(
|
|
IOUtils
|
|
.toString(
|
|
CollectorWorker.class
|
|
.getResourceAsStream(
|
|
"/eu/dnetlib/dhp/collection/collector_reporter_input_parameter.json")));
|
|
argumentParser.parseArgument(args);
|
|
|
|
final String nameNode = argumentParser.get("namenode");
|
|
log.info("nameNode is {}", nameNode);
|
|
|
|
final String mdStoreVersion = argumentParser.get("mdStoreVersion");
|
|
log.info("mdStoreVersion is {}", mdStoreVersion);
|
|
|
|
final MDStoreVersion currentVersion = MAPPER.readValue(mdStoreVersion, MDStoreVersion.class);
|
|
|
|
final String reportPath = currentVersion.getHdfsPath() + REPORT_FILE_NAME;
|
|
log.info("report path is {}", reportPath);
|
|
|
|
final Configuration conf = getHadoopConfiguration(nameNode);
|
|
CollectorPluginReport report = readHdfsFileAs(conf, reportPath, CollectorPluginReport.class);
|
|
if (Objects.isNull(report)) {
|
|
throw new CollectorException("collection report is NULL");
|
|
}
|
|
log.info("report success: {}, size: {}", report.isSuccess(), report.size());
|
|
report.forEach((k, v) -> log.info("{} - {}", k, v));
|
|
if (!report.isSuccess()) {
|
|
throw new CollectorException("collection report indicates a failure");
|
|
}
|
|
}
|
|
|
|
}
|