WIP: collectorWorker error reporting, added report messages
parent
523a6bfa97
commit
1abe6d1ad7
@ -0,0 +1,20 @@
|
||||
|
||||
package eu.dnetlib.dhp.message;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
public enum MessageType {
|
||||
|
||||
ONGOING, REPORT;
|
||||
|
||||
public MessageType from(String value) {
|
||||
return Optional
|
||||
.ofNullable(value)
|
||||
.map(StringUtils::upperCase)
|
||||
.map(MessageType::valueOf)
|
||||
.orElseThrow(() -> new IllegalArgumentException("unknown message type: " + value));
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,57 +1,39 @@
|
||||
|
||||
package eu.dnetlib.dhp.collection;
|
||||
|
||||
import static eu.dnetlib.dhp.utils.DHPUtils.*;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import eu.dnetlib.dhp.message.MessageSender;
|
||||
|
||||
public class CollectorPluginReport extends LinkedHashMap<String, String> implements Closeable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(CollectorPluginReport.class);
|
||||
|
||||
@JsonIgnore
|
||||
private Path path;
|
||||
|
||||
@JsonIgnore
|
||||
private FSDataOutputStream fos;
|
||||
|
||||
public static String SUCCESS = "success";
|
||||
private MessageSender messageSender;
|
||||
|
||||
public CollectorPluginReport() {
|
||||
}
|
||||
|
||||
public CollectorPluginReport(FileSystem fs, Path path) throws IOException {
|
||||
this.path = path;
|
||||
this.fos = fs.create(path);
|
||||
}
|
||||
|
||||
public Boolean isSuccess() {
|
||||
return containsKey(SUCCESS) && Boolean.valueOf(get(SUCCESS));
|
||||
public CollectorPluginReport(MessageSender messageSender) throws IOException {
|
||||
this.messageSender = messageSender;
|
||||
}
|
||||
|
||||
public void setSuccess(Boolean success) {
|
||||
put(SUCCESS, String.valueOf(success));
|
||||
public void ongoing(Long current, Long total) {
|
||||
messageSender.sendMessage(current, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
final String data = MAPPER.writeValueAsString(this);
|
||||
if (Objects.nonNull(fos)) {
|
||||
log.info("writing report {} to {}", data, path.toString());
|
||||
IOUtils.write(data, fos);
|
||||
populateOOZIEEnv(this);
|
||||
if (Objects.nonNull(messageSender)) {
|
||||
log.info("closing report: ");
|
||||
this.forEach((k, v) -> log.info("{} - {}", k, v));
|
||||
messageSender.sendReport(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,62 +0,0 @@
|
||||
|
||||
package eu.dnetlib.dhp.collection;
|
||||
|
||||
import static eu.dnetlib.dhp.common.Constants.REPORT_FILE_NAME;
|
||||
import static eu.dnetlib.dhp.utils.DHPUtils.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion;
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
|
||||
/**
|
||||
* CollectorWorkerReporter
|
||||
*/
|
||||
public class CollectorWorkerReporter {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(CollectorWorkerReporter.class);
|
||||
|
||||
/**
|
||||
* @param args
|
||||
*/
|
||||
public static void main(final String[] args) throws IOException, ParseException, CollectorException {
|
||||
|
||||
final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser(
|
||||
IOUtils
|
||||
.toString(
|
||||
CollectorWorker.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/collection/collector_reporter_input_parameter.json")));
|
||||
argumentParser.parseArgument(args);
|
||||
|
||||
final String nameNode = argumentParser.get("namenode");
|
||||
log.info("nameNode is {}", nameNode);
|
||||
|
||||
final String mdStoreVersion = argumentParser.get("mdStoreVersion");
|
||||
log.info("mdStoreVersion is {}", mdStoreVersion);
|
||||
|
||||
final MDStoreVersion currentVersion = MAPPER.readValue(mdStoreVersion, MDStoreVersion.class);
|
||||
|
||||
final String reportPath = currentVersion.getHdfsPath() + REPORT_FILE_NAME;
|
||||
log.info("report path is {}", reportPath);
|
||||
|
||||
final Configuration conf = getHadoopConfiguration(nameNode);
|
||||
CollectorPluginReport report = readHdfsFileAs(conf, reportPath, CollectorPluginReport.class);
|
||||
if (Objects.isNull(report)) {
|
||||
throw new CollectorException("collection report is NULL");
|
||||
}
|
||||
log.info("report success: {}, size: {}", report.isSuccess(), report.size());
|
||||
report.forEach((k, v) -> log.info("{} - {}", k, v));
|
||||
if (!report.isSuccess()) {
|
||||
throw new CollectorException("collection report indicates a failure");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
[
|
||||
{
|
||||
"paramName": "n",
|
||||
"paramLongName": "namenode",
|
||||
"paramDescription": "the Name Node URI",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "mv",
|
||||
"paramLongName": "mdStoreVersion",
|
||||
"paramDescription": "the MDStore Version bean",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
@ -1,30 +0,0 @@
|
||||
|
||||
package eu.dnetlib.dhp.collector.worker.utils;
|
||||
|
||||
import static eu.dnetlib.dhp.utils.DHPUtils.*;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import eu.dnetlib.dhp.collection.CollectorPluginReport;
|
||||
|
||||
public class CollectorPluginReportTest {
|
||||
|
||||
@Test
|
||||
public void testSerialize() throws IOException {
|
||||
CollectorPluginReport r1 = new CollectorPluginReport();
|
||||
r1.put("a", "b");
|
||||
r1.setSuccess(true);
|
||||
|
||||
String s = MAPPER.writeValueAsString(r1);
|
||||
|
||||
Assertions.assertNotNull(s);
|
||||
|
||||
CollectorPluginReport r2 = MAPPER.readValue(s, CollectorPluginReport.class);
|
||||
|
||||
Assertions.assertTrue(r2.isSuccess(), "should be true");
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue