2020-04-27 14:52:31 +02:00
|
|
|
|
2021-02-24 15:07:24 +01:00
|
|
|
package eu.dnetlib.dhp.common;
|
2020-04-10 17:53:07 +02:00
|
|
|
|
2020-01-17 15:26:21 +01:00
|
|
|
import java.io.Closeable;
|
|
|
|
import java.io.IOException;
|
2021-05-04 18:06:21 +02:00
|
|
|
import java.util.*;
|
2020-01-17 15:26:21 +01:00
|
|
|
import java.util.stream.StreamSupport;
|
2020-04-28 11:23:29 +02:00
|
|
|
|
2020-04-18 12:42:58 +02:00
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
|
import org.bson.Document;
|
2021-03-03 10:22:53 +01:00
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
2020-01-17 15:26:21 +01:00
|
|
|
|
2020-04-28 11:23:29 +02:00
|
|
|
import com.google.common.collect.Iterables;
|
2021-05-04 18:06:21 +02:00
|
|
|
import com.mongodb.*;
|
2020-04-28 11:23:29 +02:00
|
|
|
import com.mongodb.client.MongoCollection;
|
|
|
|
import com.mongodb.client.MongoDatabase;
|
2021-05-04 18:06:21 +02:00
|
|
|
import com.mongodb.client.MongoIterable;
|
2020-04-28 11:23:29 +02:00
|
|
|
|
2020-01-17 15:26:21 +01:00
|
|
|
public class MdstoreClient implements Closeable {
|
|
|
|
|
2021-03-03 10:17:16 +01:00
|
|
|
private static final Logger log = LoggerFactory.getLogger(MdstoreClient.class);
|
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
private final MongoClient client;
|
|
|
|
private final MongoDatabase db;
|
2020-04-18 12:42:58 +02:00
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
public static final String MD_ID = "mdId";
|
|
|
|
public static final String CURRENT_ID = "currentId";
|
|
|
|
public static final String EXPIRING = "expiring";
|
|
|
|
public static final String ID = "id";
|
|
|
|
public static final String LAST_READ = "lastRead";
|
|
|
|
|
|
|
|
public static final String FORMAT = "format";
|
|
|
|
public static final String LAYOUT = "layout";
|
|
|
|
public static final String INTERPRETATION = "interpretation";
|
|
|
|
|
|
|
|
public static final String BODY = "body";
|
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
private static final String COLL_METADATA = "metadata";
|
|
|
|
private static final String COLL_METADATA_MANAGER = "metadataManager";
|
2020-04-18 12:42:58 +02:00
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
public MdstoreClient(final MongoClient mongoClient, final String dbName) {
|
|
|
|
this.client = mongoClient;
|
2020-04-27 14:52:31 +02:00
|
|
|
this.db = getDb(client, dbName);
|
|
|
|
}
|
2020-04-18 12:42:58 +02:00
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
public Iterable<String> mdStoreRecords(final String mdId) {
|
|
|
|
return recordIterator(mdStore(mdId));
|
|
|
|
}
|
|
|
|
|
2021-02-24 15:07:24 +01:00
|
|
|
public MongoCollection<Document> mdStore(final String mdId) {
|
2021-05-04 18:06:21 +02:00
|
|
|
final Document mdStoreInfo = getMDStoreInfo(mdId);
|
|
|
|
final String currentId = mdStoreInfo.getString(CURRENT_ID);
|
|
|
|
log.info("reading currentId: {}", currentId);
|
2021-02-24 15:07:24 +01:00
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
return getColl(db, currentId, true);
|
|
|
|
}
|
2021-03-03 10:17:16 +01:00
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
public MdstoreTx readLock(final String mdId) {
|
2021-03-03 10:17:16 +01:00
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
final Document mdStoreInfo = getMDStoreInfo(mdId);
|
|
|
|
final List expiring = mdStoreInfo.get(EXPIRING, List.class);
|
|
|
|
final String currentId = mdStoreInfo.getString(CURRENT_ID);
|
2021-03-03 10:17:16 +01:00
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
log.info("locking collection {}", currentId);
|
|
|
|
|
|
|
|
if (expiring.size() > 0) {
|
|
|
|
for (Object value : expiring) {
|
|
|
|
final Document obj = (Document) value;
|
|
|
|
final String expiringId = (String) obj.get(ID);
|
|
|
|
if (currentId.equals(expiringId)) {
|
|
|
|
obj.put(LAST_READ, new Date());
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
final BasicDBObject readStore = new BasicDBObject();
|
|
|
|
readStore.put(ID, currentId);
|
|
|
|
readStore.put(LAST_READ, new Date());
|
|
|
|
expiring.add(readStore);
|
|
|
|
}
|
|
|
|
|
|
|
|
getColl(db, COLL_METADATA_MANAGER, true)
|
|
|
|
.findOneAndReplace(new BasicDBObject("_id", mdStoreInfo.get("_id")), mdStoreInfo);
|
|
|
|
|
|
|
|
return new MdstoreTx(this, mdId, currentId);
|
2021-02-24 15:07:24 +01:00
|
|
|
}
|
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
public void readUnlock(final String mdId, final String currentId) {
|
|
|
|
|
|
|
|
log.info("unlocking collection {}", currentId);
|
|
|
|
|
|
|
|
final Document mdStoreInfo = getMDStoreInfo(mdId);
|
|
|
|
final List<Document> expiring = mdStoreInfo.get(EXPIRING, List.class);
|
|
|
|
|
|
|
|
expiring
|
|
|
|
.stream()
|
|
|
|
.filter(d -> currentId.equals(d.getString(ID)))
|
|
|
|
.findFirst()
|
|
|
|
.ifPresent(expired -> expiring.remove(expired));
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Retrieves from the MDStore mongoDB database a snapshot of the [mdID, currentID] pairs.
|
|
|
|
* @param mdFormat
|
|
|
|
* @param mdLayout
|
|
|
|
* @param mdInterpretation
|
|
|
|
* @return an HashMap of the mdID -> currentID associations.
|
|
|
|
*/
|
2020-04-27 14:52:31 +02:00
|
|
|
public Map<String, String> validCollections(
|
|
|
|
final String mdFormat, final String mdLayout, final String mdInterpretation) {
|
2020-04-18 12:42:58 +02:00
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
final Map<String, String> transactions = new HashMap<>();
|
|
|
|
for (final Document entry : getColl(db, COLL_METADATA_MANAGER, true).find()) {
|
2021-05-04 18:06:21 +02:00
|
|
|
final String mdId = entry.getString(MD_ID);
|
|
|
|
final String currentId = entry.getString(CURRENT_ID);
|
2020-04-27 14:52:31 +02:00
|
|
|
if (StringUtils.isNoneBlank(mdId, currentId)) {
|
|
|
|
transactions.put(mdId, currentId);
|
|
|
|
}
|
|
|
|
}
|
2020-04-18 12:42:58 +02:00
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
final Map<String, String> res = new HashMap<>();
|
|
|
|
for (final Document entry : getColl(db, COLL_METADATA, true).find()) {
|
2021-05-04 18:06:21 +02:00
|
|
|
if (entry.getString(FORMAT).equals(mdFormat)
|
|
|
|
&& entry.getString(LAYOUT).equals(mdLayout)
|
|
|
|
&& entry.getString(INTERPRETATION).equals(mdInterpretation)
|
|
|
|
&& transactions.containsKey(entry.getString(MD_ID))) {
|
|
|
|
res.put(entry.getString(MD_ID), transactions.get(entry.getString(MD_ID)));
|
2020-04-27 14:52:31 +02:00
|
|
|
}
|
|
|
|
}
|
2020-04-18 12:42:58 +02:00
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
return res;
|
|
|
|
}
|
2020-04-18 12:42:58 +02:00
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
private MongoDatabase getDb(final MongoClient client, final String dbName) {
|
|
|
|
if (!Iterables.contains(client.listDatabaseNames(), dbName)) {
|
|
|
|
final String err = String.format("Database '%s' not found in %s", dbName, client.getAddress());
|
|
|
|
log.warn(err);
|
|
|
|
throw new RuntimeException(err);
|
|
|
|
}
|
|
|
|
return client.getDatabase(dbName);
|
|
|
|
}
|
2020-04-18 12:42:58 +02:00
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
private MongoCollection<Document> getColl(
|
|
|
|
final MongoDatabase db, final String collName, final boolean abortIfMissing) {
|
|
|
|
if (!Iterables.contains(db.listCollectionNames(), collName)) {
|
2021-05-04 18:06:21 +02:00
|
|
|
final String err = String.format("Missing collection '%s' in database '%s'", collName, db.getName());
|
2020-04-27 14:52:31 +02:00
|
|
|
log.warn(err);
|
|
|
|
if (abortIfMissing) {
|
|
|
|
throw new RuntimeException(err);
|
|
|
|
} else {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return db.getCollection(collName);
|
|
|
|
}
|
2020-04-27 14:45:40 +02:00
|
|
|
|
2021-05-04 18:06:21 +02:00
|
|
|
private Document getMDStoreInfo(final String mdId) {
|
|
|
|
return Optional
|
|
|
|
.ofNullable(getColl(db, COLL_METADATA_MANAGER, true))
|
|
|
|
.map(metadataManager -> {
|
|
|
|
BasicDBObject query = (BasicDBObject) QueryBuilder.start(MD_ID).is(mdId).get();
|
|
|
|
log.info("querying current mdId: {}", query.toJson());
|
|
|
|
return Optional
|
|
|
|
.ofNullable(metadataManager.find(query))
|
|
|
|
.map(MongoIterable::first)
|
|
|
|
.orElseThrow(() -> new IllegalArgumentException("cannot find current mdstore id for: " + mdId));
|
|
|
|
})
|
|
|
|
.orElseThrow(() -> new IllegalStateException("missing collection " + COLL_METADATA_MANAGER));
|
|
|
|
}
|
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
public Iterable<String> listRecords(final String collName) {
|
2021-05-04 18:06:21 +02:00
|
|
|
return recordIterator(getColl(db, collName, false));
|
|
|
|
}
|
|
|
|
|
|
|
|
private Iterable<String> recordIterator(MongoCollection<Document> coll) {
|
2020-04-27 14:52:31 +02:00
|
|
|
return coll == null
|
|
|
|
? new ArrayList<>()
|
|
|
|
: () -> StreamSupport
|
|
|
|
.stream(coll.find().spliterator(), false)
|
2021-05-04 18:06:21 +02:00
|
|
|
.filter(e -> e.containsKey(BODY))
|
|
|
|
.map(e -> e.getString(BODY))
|
2020-04-27 14:52:31 +02:00
|
|
|
.iterator();
|
|
|
|
}
|
2020-04-27 14:45:40 +02:00
|
|
|
|
2020-04-27 14:52:31 +02:00
|
|
|
@Override
|
|
|
|
public void close() throws IOException {
|
|
|
|
client.close();
|
|
|
|
}
|
2020-01-17 15:26:21 +01:00
|
|
|
}
|