2021-01-28 09:51:17 +01:00
|
|
|
|
2021-02-12 12:31:02 +01:00
|
|
|
package eu.dnetlib.dhp.collection;
|
2021-01-28 09:51:17 +01:00
|
|
|
|
2021-02-08 12:19:38 +01:00
|
|
|
import static eu.dnetlib.dhp.common.Constants.SEQUENCE_FILE_NAME;
|
2021-02-05 19:18:05 +01:00
|
|
|
|
2021-01-28 09:51:17 +01:00
|
|
|
import java.io.IOException;
|
2021-02-12 12:31:02 +01:00
|
|
|
import java.util.Optional;
|
2021-01-28 09:51:17 +01:00
|
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
|
2021-02-12 12:31:02 +01:00
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
|
import org.apache.hadoop.fs.FileSystem;
|
2021-01-28 09:51:17 +01:00
|
|
|
import org.apache.hadoop.fs.Path;
|
|
|
|
import org.apache.hadoop.io.IntWritable;
|
|
|
|
import org.apache.hadoop.io.SequenceFile;
|
|
|
|
import org.apache.hadoop.io.Text;
|
2021-02-12 12:31:02 +01:00
|
|
|
import org.apache.hadoop.io.compress.DeflateCodec;
|
2021-01-28 09:51:17 +01:00
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
2021-02-05 19:18:05 +01:00
|
|
|
import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion;
|
2021-01-28 09:51:17 +01:00
|
|
|
import eu.dnetlib.dhp.collection.plugin.CollectorPlugin;
|
2021-02-12 12:31:02 +01:00
|
|
|
import eu.dnetlib.dhp.collection.plugin.mongodb.MongoDbCollectorPlugin;
|
|
|
|
import eu.dnetlib.dhp.collection.plugin.mongodb.MongoDbDumpCollectorPlugin;
|
|
|
|
import eu.dnetlib.dhp.collection.plugin.oai.OaiCollectorPlugin;
|
2021-02-05 19:18:05 +01:00
|
|
|
import eu.dnetlib.dhp.message.MessageSender;
|
2021-01-28 09:51:17 +01:00
|
|
|
|
|
|
|
public class CollectorWorker {
|
|
|
|
|
|
|
|
private static final Logger log = LoggerFactory.getLogger(CollectorWorker.class);
|
|
|
|
|
|
|
|
private final ApiDescriptor api;
|
|
|
|
|
2021-02-12 12:31:02 +01:00
|
|
|
private final FileSystem fileSystem;
|
2021-01-28 09:51:17 +01:00
|
|
|
|
2021-02-05 19:18:05 +01:00
|
|
|
private final MDStoreVersion mdStoreVersion;
|
2021-01-28 09:51:17 +01:00
|
|
|
|
2021-02-05 19:18:05 +01:00
|
|
|
private final HttpClientParams clientParams;
|
|
|
|
|
|
|
|
private final CollectorPluginReport report;
|
2021-02-04 15:51:15 +01:00
|
|
|
|
2021-02-05 19:18:05 +01:00
|
|
|
private final MessageSender messageSender;
|
2021-02-04 15:51:15 +01:00
|
|
|
|
2021-01-28 09:51:17 +01:00
|
|
|
public CollectorWorker(
|
|
|
|
final ApiDescriptor api,
|
2021-02-12 12:31:02 +01:00
|
|
|
final FileSystem fileSystem,
|
2021-02-05 19:18:05 +01:00
|
|
|
final MDStoreVersion mdStoreVersion,
|
|
|
|
final HttpClientParams clientParams,
|
|
|
|
final MessageSender messageSender,
|
|
|
|
final CollectorPluginReport report) {
|
2021-01-28 09:51:17 +01:00
|
|
|
this.api = api;
|
2021-02-12 12:31:02 +01:00
|
|
|
this.fileSystem = fileSystem;
|
2021-02-05 19:18:05 +01:00
|
|
|
this.mdStoreVersion = mdStoreVersion;
|
|
|
|
this.clientParams = clientParams;
|
2021-02-04 15:51:15 +01:00
|
|
|
this.messageSender = messageSender;
|
2021-02-05 19:18:05 +01:00
|
|
|
this.report = report;
|
2021-01-28 09:51:17 +01:00
|
|
|
}
|
|
|
|
|
2021-02-05 19:18:05 +01:00
|
|
|
public void collect() throws UnknownCollectorPluginException, CollectorException, IOException {
|
2021-02-03 12:33:41 +01:00
|
|
|
|
2021-02-05 19:18:05 +01:00
|
|
|
final String outputPath = mdStoreVersion.getHdfsPath() + SEQUENCE_FILE_NAME;
|
|
|
|
log.info("outputPath path is {}", outputPath);
|
2021-02-03 12:33:41 +01:00
|
|
|
|
2021-02-12 12:31:02 +01:00
|
|
|
final CollectorPlugin plugin = getCollectorPlugin();
|
2021-02-03 12:33:41 +01:00
|
|
|
final AtomicInteger counter = new AtomicInteger(0);
|
2021-02-04 15:51:15 +01:00
|
|
|
|
2021-02-03 12:33:41 +01:00
|
|
|
try (SequenceFile.Writer writer = SequenceFile
|
|
|
|
.createWriter(
|
2021-02-12 12:31:02 +01:00
|
|
|
fileSystem.getConf(),
|
2021-02-05 19:18:05 +01:00
|
|
|
SequenceFile.Writer.file(new Path(outputPath)),
|
2021-02-03 12:33:41 +01:00
|
|
|
SequenceFile.Writer.keyClass(IntWritable.class),
|
2021-02-08 18:06:25 +01:00
|
|
|
SequenceFile.Writer.valueClass(Text.class),
|
2021-02-12 12:31:02 +01:00
|
|
|
SequenceFile.Writer.compression(SequenceFile.CompressionType.BLOCK, new DeflateCodec()))) {
|
2021-02-03 12:33:41 +01:00
|
|
|
final IntWritable key = new IntWritable(counter.get());
|
|
|
|
final Text value = new Text();
|
|
|
|
plugin
|
2021-02-05 19:18:05 +01:00
|
|
|
.collect(api, report)
|
2021-02-03 12:33:41 +01:00
|
|
|
.forEach(
|
|
|
|
content -> {
|
|
|
|
key.set(counter.getAndIncrement());
|
2021-02-05 19:18:05 +01:00
|
|
|
if (counter.get() % 500 == 0)
|
2021-02-04 15:51:15 +01:00
|
|
|
messageSender.sendMessage(counter.longValue(), null);
|
2021-02-03 12:33:41 +01:00
|
|
|
value.set(content);
|
|
|
|
try {
|
|
|
|
writer.append(key, value);
|
2021-02-05 19:18:05 +01:00
|
|
|
} catch (Throwable e) {
|
|
|
|
report.put(e.getClass().getName(), e.getMessage());
|
|
|
|
log.warn("setting report to failed");
|
|
|
|
report.setSuccess(false);
|
2021-02-03 12:33:41 +01:00
|
|
|
throw new RuntimeException(e);
|
|
|
|
}
|
|
|
|
});
|
2021-02-05 19:18:05 +01:00
|
|
|
} catch (Throwable e) {
|
|
|
|
report.put(e.getClass().getName(), e.getMessage());
|
|
|
|
log.warn("setting report to failed");
|
|
|
|
report.setSuccess(false);
|
2021-02-03 12:33:41 +01:00
|
|
|
} finally {
|
2021-02-05 19:18:05 +01:00
|
|
|
messageSender.sendMessage(counter.longValue(), counter.longValue());
|
2021-01-28 09:51:17 +01:00
|
|
|
}
|
|
|
|
}
|
2021-02-05 19:18:05 +01:00
|
|
|
|
2021-02-12 12:31:02 +01:00
|
|
|
private CollectorPlugin getCollectorPlugin() throws UnknownCollectorPluginException {
|
|
|
|
switch (StringUtils.lowerCase(StringUtils.trim(api.getProtocol()))) {
|
|
|
|
case "oai":
|
|
|
|
return new OaiCollectorPlugin(clientParams);
|
|
|
|
case "other":
|
|
|
|
final String plugin = Optional
|
|
|
|
.ofNullable(api.getParams().get("other_plugin_type"))
|
|
|
|
.orElseThrow(() -> new UnknownCollectorPluginException("other_plugin_type"));
|
|
|
|
|
|
|
|
switch (plugin) {
|
|
|
|
case "mdstore_mongodb_dump":
|
|
|
|
return new MongoDbDumpCollectorPlugin(fileSystem);
|
|
|
|
case "mdstore_mongodb":
|
|
|
|
return new MongoDbCollectorPlugin();
|
|
|
|
default:
|
|
|
|
throw new UnknownCollectorPluginException("Unknown plugin type: " + plugin);
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
throw new UnknownCollectorPluginException("Unknown protocol: " + api.getProtocol());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-28 09:51:17 +01:00
|
|
|
}
|