package eu.dnetlib.dhp.oa.graph.raw; import java.io.Closeable; import java.io.IOException; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.MdstoreClient; import eu.dnetlib.dhp.common.MdstoreTx; import eu.dnetlib.dhp.oa.graph.raw.common.AbstractMigrationApplication; public class MigrateMongoMdstoresApplication extends AbstractMigrationApplication implements Closeable { private static final Logger log = LoggerFactory.getLogger(MigrateMongoMdstoresApplication.class); private final MdstoreClient mdstoreClient; public static void main(final String[] args) throws Exception { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils .toString( MigrateMongoMdstoresApplication.class .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/migrate_mongo_mstores_parameters.json"))); parser.parseArgument(args); final String mongoBaseUrl = parser.get("mongoBaseUrl"); final String mongoDb = parser.get("mongoDb"); final String mdFormat = parser.get("mdFormat"); final String mdLayout = parser.get("mdLayout"); final String mdInterpretation = parser.get("mdInterpretation"); final String hdfsPath = parser.get("hdfsPath"); final MongoClient mongoClient = new MongoClient(new MongoClientURI(mongoBaseUrl)); try (MigrateMongoMdstoresApplication app = new MigrateMongoMdstoresApplication(hdfsPath, mongoClient, mongoDb)) { app.execute(mdFormat, mdLayout, mdInterpretation); } } public MigrateMongoMdstoresApplication( final String hdfsPath, final MongoClient mongoClient, final String mongoDb) throws Exception { super(hdfsPath); this.mdstoreClient = new MdstoreClient(mongoClient, mongoDb); } public void execute(final String format, final String layout, final String interpretation) throws IOException { final Map colls = mdstoreClient.validCollections(format, layout, interpretation); log.info("Found {} mdstores", colls.size()); for (final Entry entry : colls.entrySet()) { log.info("Processing mdstore {}", entry.getKey()); final String mdID = entry.getKey(); try (final MdstoreTx tx = mdstoreClient.readLock(mdID)) { log.info("locked collection {}", tx.getCurrentId()); for (final String xml : tx) { emit(xml, String.format("%s-%s-%s", format, layout, interpretation)); } } } } @Override public void close() throws IOException { super.close(); mdstoreClient.close(); } }