You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
319 lines
13 KiB
Java
319 lines
13 KiB
Java
/**
|
|
*
|
|
*/
|
|
package org.gcube.contentmanagement.blobstorage.transport.backend.operation;
|
|
|
|
import java.io.InputStream;
|
|
import java.net.UnknownHostException;
|
|
|
|
import org.bson.types.ObjectId;
|
|
import org.gcube.contentmanagement.blobstorage.resource.MemoryType;
|
|
import org.gcube.contentmanagement.blobstorage.resource.MyFile;
|
|
import org.gcube.contentmanagement.blobstorage.resource.OperationDefinition.LOCAL_RESOURCE;
|
|
import org.gcube.contentmanagement.blobstorage.resource.OperationDefinition.OPERATION;
|
|
import org.gcube.contentmanagement.blobstorage.resource.OperationDefinition.REMOTE_RESOURCE;
|
|
import org.gcube.contentmanagement.blobstorage.service.operation.Monitor;
|
|
import org.gcube.contentmanagement.blobstorage.service.operation.Operation;
|
|
import org.gcube.contentmanagement.blobstorage.service.operation.SoftCopy;
|
|
import org.gcube.contentmanagement.blobstorage.transport.backend.MongoIOManager;
|
|
import org.gcube.contentmanagement.blobstorage.transport.backend.MongoOperationManager;
|
|
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
|
|
import org.gcube.contentmanagement.blobstorage.transport.backend.util.Costants;
|
|
import org.gcube.contentmanagement.blobstorage.transport.backend.util.DateUtils;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import com.mongodb.BasicDBObject;
|
|
import com.mongodb.DBCollection;
|
|
import com.mongodb.DBObject;
|
|
import com.mongodb.gridfs.GridFSDBFile;
|
|
|
|
/**
|
|
* @author Roberto Cirillo (ISTI-CNR) 2018
|
|
*
|
|
*/
|
|
public class SoftCopyOperator extends SoftCopy {
|
|
|
|
Logger logger=LoggerFactory.getLogger(SoftCopyOperator.class);
|
|
private MemoryType memoryType;
|
|
private MongoIOManager mongoPrimaryInstance;
|
|
private MyFile resource;
|
|
/**
|
|
* @param server
|
|
* @param user
|
|
* @param pwd
|
|
* @param bucket
|
|
* @param monitor
|
|
* @param isChunk
|
|
* @param backendType
|
|
* @param dbs
|
|
*/
|
|
public SoftCopyOperator(String[] server, String user, String pwd, String bucket, Monitor monitor, boolean isChunk,
|
|
String backendType, String[] dbs) {
|
|
super(server, user, pwd, bucket, monitor, isChunk, backendType, dbs);
|
|
// TODO Auto-generated constructor stub
|
|
}
|
|
|
|
@Override
|
|
public String execute(MongoIOManager mongoPrimaryInstance, MyFile resource, String sourcePath, String destinationPath)
|
|
throws UnknownHostException {
|
|
REMOTE_RESOURCE remoteResourceIdentifier=resource.getOperation().getRemoteResource();
|
|
LOCAL_RESOURCE localResourceIdentifier=resource.getOperation().getLocalResource();
|
|
String source=null;
|
|
if(localResourceIdentifier.equals(LOCAL_RESOURCE.ID))
|
|
source=resource.getId();
|
|
else
|
|
source=sourcePath;
|
|
String destination=null;
|
|
if(remoteResourceIdentifier.equals(REMOTE_RESOURCE.ID))
|
|
destination=resource.getId();
|
|
else
|
|
destination=destinationPath;
|
|
if(resource!=null){
|
|
String dir=((MyFile)resource).getRemoteDir();
|
|
String name=((MyFile)resource).getName();
|
|
setMemoryType(((MyFile)resource).getGcubeMemoryType());
|
|
}
|
|
setMongoPrimaryInstance(mongoPrimaryInstance);
|
|
ObjectId mapId=null;
|
|
GridFSDBFile destObject=null;
|
|
logger.debug("softCopy operation on Mongo backend, parameters: source path: "+source+" destination path: "+destination);
|
|
if((source != null) && (!source.isEmpty())){
|
|
GridFSDBFile sourceObject = mongoPrimaryInstance.retrieveRemoteDescriptor(source, remoteResourceIdentifier, true);
|
|
if(sourceObject != null){
|
|
// GridFSDBFile originalObject=sourceObject;
|
|
// if it contains a link field, then I'm going to retrieve the related payload
|
|
sourceObject = mongoPrimaryInstance.retrieveLinkPayload(sourceObject);
|
|
ObjectId sourceId=(ObjectId)sourceObject.getId();
|
|
InputStream is= sourceObject.getInputStream();
|
|
resource.setInputStream(is);
|
|
GridFSDBFile dest = null;
|
|
if((destination == null) || (destination.isEmpty())){
|
|
// if the destination param is null, the destination object will be filled with values extracted from sourceObject
|
|
if(sourceId==null) throw new RemoteBackendException("source object not found: "+source);
|
|
destination = fillGenericDestinationFields(resource, sourceId);
|
|
logger.warn("SoftCopy without destination parameter. The operation will be executed with the following destination path "+destination);
|
|
}else{
|
|
// check if the destination is a dir or a file and if the destination exist
|
|
dest = mongoPrimaryInstance.retrieveRemoteDescriptor(destination, remoteResourceIdentifier, false);//gfs.findOne(destination);
|
|
}
|
|
// check if the destination is a dir or a file and if the destination exist
|
|
// GridFSDBFile dest = mongoPrimaryInstance.retrieveRemoteDescriptor(destination, remoteResourceIdentifier, false);//gfs.findOne(destination);
|
|
// GridFSInputFile destinationFile=mongoPrimaryInstance.createGFSFileObject(is, resource.getWriteConcern(), resource.getReadPreference());//gfs.createFile(is);
|
|
ObjectId removedId=null;
|
|
// if the destination location is not empty
|
|
if (dest != null){
|
|
// remove the destination file. The third parameter to true replace the file otherwise the remote id is returned
|
|
if(resource.isReplace()){
|
|
removedId = mongoPrimaryInstance.removeFile(resource, null, resource.isReplace(), null, dest);
|
|
}else{
|
|
return dest.getId().toString();
|
|
}
|
|
}
|
|
// get metacollection instance
|
|
DBCollection metaCollectionInstance = getMetaCollection();
|
|
String md5=sourceObject.getMD5();
|
|
// check if the payload is already present on backend
|
|
ObjectId md5Id=getDuplicatesMap(md5);
|
|
// check if the source object is already a map
|
|
if(isMap(sourceObject)){
|
|
logger.debug("the sourceObject with the following id: "+mapId+" is already a map");
|
|
mapId=sourceId;
|
|
// then it's needed to add only the destObject to the map
|
|
//first: create link object to destination place
|
|
DBObject newObject=createNewLinkObject(resource, sourceObject, destination, metaCollectionInstance, md5, mapId, removedId);
|
|
destObject = mongoPrimaryInstance.retrieveRemoteDescriptor(destination, remoteResourceIdentifier, true);
|
|
// second: add the new object to the map
|
|
mapId = addToDuplicateMap(metaCollectionInstance, mapId, destObject);
|
|
// if the payload is already present on backend
|
|
}else if(md5Id!=null){
|
|
mapId=md5Id;
|
|
logger.debug("retrieved md5 on backend with the following id: "+mapId);
|
|
mapId = addToDuplicateMap(metaCollectionInstance, mapId, sourceObject);
|
|
DBObject newObject=createNewLinkObject(resource, sourceObject, destination, metaCollectionInstance, md5, mapId, removedId);
|
|
destObject = mongoPrimaryInstance.retrieveRemoteDescriptor(destination, remoteResourceIdentifier, true);
|
|
mapId = addToDuplicateMap(metaCollectionInstance, mapId, destObject);
|
|
}else{
|
|
// no map present no md5 present
|
|
mapId = createNewDuplicatesMap(metaCollectionInstance, resource, sourceObject, destination, sourceId);
|
|
mapId = addToDuplicateMap(metaCollectionInstance, mapId, sourceObject);
|
|
DBObject newObject=createNewLinkObject(resource, sourceObject,destination, metaCollectionInstance, md5, mapId, removedId);
|
|
destObject = mongoPrimaryInstance.retrieveRemoteDescriptor(destination, remoteResourceIdentifier, true);
|
|
mapId = addToDuplicateMap(metaCollectionInstance, mapId, destObject);
|
|
}
|
|
if(logger.isDebugEnabled())
|
|
logger.debug("mapId created/updated: "+mapId);
|
|
mongoPrimaryInstance.close();
|
|
}else{
|
|
mongoPrimaryInstance.close();
|
|
throw new RemoteBackendException(" the source path is wrong. There isn't a file at "+source);
|
|
}
|
|
}else throw new RemoteBackendException("Invalid arguments: source "+source+" destination "+destination);
|
|
// return mapId.toString();
|
|
return destObject.getId().toString();
|
|
}
|
|
|
|
private String fillGenericDestinationFields(MyFile resource, ObjectId souceId) {
|
|
String destination;
|
|
destination=resource.getRootPath()+souceId;
|
|
resource.setName(souceId.toString());
|
|
resource.setRemoteDir(resource.getRootPath());
|
|
return destination;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param resource
|
|
* @param bucket destinationPath
|
|
* @param dir destination directory
|
|
* @param name name of the new file
|
|
* @param oldId id of the file was present in the destination place
|
|
* @return id of the new map
|
|
* @throws UnknownHostException
|
|
*/
|
|
private ObjectId createNewDuplicatesMap(DBCollection metaCollectionInstance, Object resource, GridFSDBFile sourceObject, String bucket, ObjectId sourceId) throws UnknownHostException {
|
|
ObjectId id = null;
|
|
String dir= ((MyFile)resource).getRemoteDir();
|
|
// create new dir (is it really needed in case of map object?)
|
|
if((dir !=null && !dir.isEmpty()) && (bucket !=null && !bucket.isEmpty())){
|
|
getMongoPrimaryInstance().buildDirTree(getMongoPrimaryInstance().getMetaDataCollection(null), dir);
|
|
}
|
|
// create new map object
|
|
id= createNewObjectMap(metaCollectionInstance, (MyFile)resource, sourceObject, sourceId);
|
|
return id;
|
|
}
|
|
|
|
private ObjectId createNewObjectMap(DBCollection metaCollectionInstance, MyFile resource, GridFSDBFile source, ObjectId sourceId) throws UnknownHostException {
|
|
String md5=source.getMD5();
|
|
// set type of object
|
|
DBObject document=new BasicDBObject("type", "map");
|
|
// initialize count field to 0
|
|
document.put("count", 0);
|
|
ObjectId id=new ObjectId();
|
|
document.put("_id", id);
|
|
logger.debug("generated id for new map"+id);
|
|
document=fillCommonfields(document, resource, source, metaCollectionInstance, md5);
|
|
// update chunks collection
|
|
getMongoPrimaryInstance().updateChunksCollection(sourceId, id);
|
|
return id;
|
|
}
|
|
|
|
private DBObject createNewLinkObject(MyFile resource, GridFSDBFile sourceObject, String destination, DBCollection metaCollectionInstance, String md5, ObjectId mapId, ObjectId newId){
|
|
DBObject document=new BasicDBObject("type", "file");
|
|
document.put("filename", destination);
|
|
document.put("name", resource.getName());
|
|
document.put("dir", resource.getRemoteDir());
|
|
document.put("owner", resource.getOwner());
|
|
document.put(Costants.LINK_IDENTIFIER, mapId.toString());
|
|
ObjectId id=null;
|
|
if(newId == null){
|
|
id=new ObjectId();
|
|
logger.debug("generated id for new object link"+id);
|
|
}else{
|
|
id=newId;
|
|
logger.debug("restored id for new object link"+id);
|
|
}
|
|
document.put("_id", id);
|
|
|
|
return fillCommonfields(document, resource, sourceObject, metaCollectionInstance, md5);
|
|
}
|
|
|
|
private DBObject fillCommonfields(DBObject document, MyFile resource, GridFSDBFile sourceObject, DBCollection metaCollectionInstance, String md5) {
|
|
document.put("mimetype", ((MyFile)resource).getMimeType());
|
|
document.put("creationTime", DateUtils.now("dd MM yyyy 'at' hh:mm:ss z"));
|
|
document.put("md5", md5);
|
|
document.put("length", sourceObject.getLength());
|
|
// set chunkSize inherited from original object
|
|
document.put("chunkSize", sourceObject.getChunkSize());
|
|
metaCollectionInstance.insert(document);
|
|
metaCollectionInstance.save(document);
|
|
return document;
|
|
}
|
|
|
|
private DBCollection getMetaCollection() throws UnknownHostException {
|
|
DBCollection metaCollectionInstance=null;
|
|
if(!(getMemoryType() == MemoryType.VOLATILE))
|
|
metaCollectionInstance=mongoPrimaryInstance.getMetaDataCollection(mongoPrimaryInstance.getConnectionDB(MongoOperationManager.getPrimaryCollectionName(), true));
|
|
else
|
|
metaCollectionInstance=mongoPrimaryInstance.getMetaDataCollection(mongoPrimaryInstance.getConnectionDB(MongoOperationManager.getPrimaryCollectionName(), false));
|
|
return metaCollectionInstance;
|
|
}
|
|
|
|
private ObjectId addToDuplicateMap(DBCollection metaCollectionInstance, ObjectId mapId, GridFSDBFile f) throws UnknownHostException {
|
|
f.put(Costants.LINK_IDENTIFIER, mapId.toString());
|
|
mongoPrimaryInstance.updateCommonFields(f, getResource(), OPERATION.SOFT_COPY);
|
|
f.save();
|
|
incrementCountField(metaCollectionInstance, mapId);
|
|
return mapId;
|
|
}
|
|
|
|
private void incrementCountField(DBCollection metaCollectionInstance, ObjectId mapId) throws UnknownHostException {
|
|
logger.info("increment count field on"+mapId+ " object map");
|
|
BasicDBObject searchQuery= new BasicDBObject();
|
|
searchQuery.put("_id" ,mapId);
|
|
DBObject mapObject=mongoPrimaryInstance.findCollectionObject(metaCollectionInstance, searchQuery);
|
|
// BasicDBObject updateObject= new BasicDBObject().append("$inc",new BasicDBObject().append("count", 1));;
|
|
int count=(int)mapObject.get("count");
|
|
count++;
|
|
mapObject.put("count", count);
|
|
// metaCollectionInstance.update(mapObject, updateObject);
|
|
metaCollectionInstance.save(mapObject);
|
|
}
|
|
|
|
private ObjectId getDuplicatesMap(String md5){
|
|
ObjectId id= checkMd5(md5);
|
|
return id;
|
|
}
|
|
|
|
/**
|
|
* @param sourceObject
|
|
* @return
|
|
*/
|
|
private boolean isMap(GridFSDBFile sourceObject) {
|
|
String type=sourceObject.get("type").toString();
|
|
if(type.equals("map"))
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
* Check if the backend already has the payload
|
|
* @param md5 string of the file
|
|
* @return the ObjectID of the md5 file found on the backend, else null
|
|
*/
|
|
private ObjectId checkMd5(String md5) {
|
|
// TODO Auto-generated method stub
|
|
return null;
|
|
}
|
|
|
|
public MemoryType getMemoryType() {
|
|
return memoryType;
|
|
}
|
|
|
|
public void setMemoryType(MemoryType memoryType) {
|
|
this.memoryType = memoryType;
|
|
}
|
|
|
|
public MongoIOManager getMongoPrimaryInstance() {
|
|
return mongoPrimaryInstance;
|
|
}
|
|
|
|
public void setMongoPrimaryInstance(MongoIOManager mongoPrimaryInstance) {
|
|
this.mongoPrimaryInstance = mongoPrimaryInstance;
|
|
}
|
|
|
|
public MyFile getResource() {
|
|
return resource;
|
|
}
|
|
|
|
public void setResource(MyFile resource) {
|
|
this.resource = resource;
|
|
}
|
|
|
|
|
|
|
|
}
|