117 lines
4.1 KiB
Java
117 lines
4.1 KiB
Java
|
|
package eu.dnetlib.dhp.migration;
|
|
|
|
import static eu.dnetlib.dhp.common.Constants.DNET_MESSAGE_MGR_URL;
|
|
import static eu.dnetlib.dhp.utils.DHPUtils.MAPPER;
|
|
import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration;
|
|
|
|
import java.io.IOException;
|
|
import java.util.HashMap;
|
|
import java.util.Optional;
|
|
import java.util.Spliterator;
|
|
import java.util.Spliterators;
|
|
import java.util.stream.StreamSupport;
|
|
|
|
import org.apache.commons.cli.ParseException;
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.hadoop.fs.FileSystem;
|
|
import org.bson.Document;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import com.mongodb.client.MongoCollection;
|
|
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
import eu.dnetlib.dhp.collection.ApiDescriptor;
|
|
import eu.dnetlib.dhp.collection.CollectorWorker;
|
|
import eu.dnetlib.dhp.collection.CollectorWorkerApplication;
|
|
import eu.dnetlib.dhp.collection.UnknownCollectorPluginException;
|
|
import eu.dnetlib.dhp.common.MdstoreClient;
|
|
import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
|
|
import eu.dnetlib.dhp.common.collection.CollectorException;
|
|
import eu.dnetlib.dhp.common.collection.HttpClientParams;
|
|
import eu.dnetlib.dhp.message.MessageSender;
|
|
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
|
|
|
|
public class MongoDbMDStoreCollectionReader {
|
|
|
|
private static final Logger log = LoggerFactory.getLogger(MongoDbMDStoreCollectionReader.class);
|
|
|
|
private final FileSystem fileSystem;
|
|
|
|
public MongoDbMDStoreCollectionReader(FileSystem fileSystem) {
|
|
this.fileSystem = fileSystem;
|
|
}
|
|
|
|
/**
|
|
* @param args
|
|
*/
|
|
public static void main(final String[] args)
|
|
throws ParseException, IOException, UnknownCollectorPluginException, CollectorException {
|
|
|
|
final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser(
|
|
IOUtils
|
|
.toString(
|
|
MongoDbMDStoreCollectionReader.class
|
|
.getResourceAsStream(
|
|
"/eu/dnetlib/dhp/migration/native_records_migration_input_parameter.json")));
|
|
argumentParser.parseArgument(args);
|
|
|
|
log.info("Java Xmx: {}m", Runtime.getRuntime().maxMemory() / (1024 * 1024));
|
|
|
|
final String hdfsuri = argumentParser.get("namenode");
|
|
log.info("hdfsURI is {}", hdfsuri);
|
|
|
|
final String mongoBaseUrl = Optional
|
|
.ofNullable(argumentParser.get("mongoDbUri"))
|
|
.orElseThrow(
|
|
() -> new IOException("missing mongodb baseUrl"));
|
|
log.info("mongoBaseUrl: {}", mongoBaseUrl);
|
|
|
|
final String dbName = Optional
|
|
.ofNullable(argumentParser.get("mongoDbName"))
|
|
.orElseThrow(() -> new IOException("missing parameter 'mongoDbName'"));
|
|
log.info("dbName: {}", dbName);
|
|
|
|
final String mdId = Optional
|
|
.ofNullable(argumentParser.get("sourceMDStoreId"))
|
|
.orElseThrow(() -> new IOException("missing parameter 'sourceMDStoreId'"));
|
|
log.info("sourceMDStoreId: {}", mdId);
|
|
|
|
final String mdStoreVersion = argumentParser.get("mdStoreVersion");
|
|
log.info("mdStoreVersion is {}", mdStoreVersion);
|
|
|
|
final String dnetMessageManagerURL = argumentParser.get(DNET_MESSAGE_MGR_URL);
|
|
log.info("dnetMessageManagerURL is {}", dnetMessageManagerURL);
|
|
|
|
final String workflowId = argumentParser.get("workflowId");
|
|
log.info("workflowId is {}", workflowId);
|
|
|
|
final FileSystem fileSystem = FileSystem.get(getHadoopConfiguration(hdfsuri));
|
|
|
|
new MongoDbMDStoreCollectionReader(fileSystem)
|
|
.run(mongoBaseUrl, dbName, mdId, mdStoreVersion, dnetMessageManagerURL, workflowId);
|
|
}
|
|
|
|
protected void run(String mongoBaseUrl, String dbName, String mdId, String mdStoreVersion,
|
|
String dnetMessageManagerURL, String workflowId)
|
|
throws IOException, UnknownCollectorPluginException, CollectorException {
|
|
|
|
final ApiDescriptor api = new ApiDescriptor();
|
|
api.setProtocol("other");
|
|
api.setBaseUrl(mongoBaseUrl);
|
|
api.getParams().put("other_plugin_type", "mdstore_mongodb");
|
|
api.getParams().put("mongodb_dbname", dbName);
|
|
api.getParams().put("mdstore_id", mdId);
|
|
|
|
final MDStoreVersion currentVersion = MAPPER.readValue(mdStoreVersion, MDStoreVersion.class);
|
|
final MessageSender ms = new MessageSender(dnetMessageManagerURL, workflowId);
|
|
|
|
try (AggregatorReport report = new AggregatorReport(ms)) {
|
|
|
|
new CollectorWorker(api, fileSystem, currentVersion, null, report).collect();
|
|
}
|
|
}
|
|
|
|
}
|