gCubeSystem
/
storage-manager-trigger
16
0
Fork 0
Code Issues Releases Wiki Activity

bugFix on volume calculation 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/storage-manager-trigger@100875 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
roberto.cirillo 2014-10-22 15:47:24 +00:00
parent d2ef6cf128
commit 6a578aa947
9 changed files with 99 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/org/gcube/contentmanager/storageserver/consumer/FolderAccountingConsumer.java
View File

@ -8,7 +8,7 @@ import org.gcube.contentmanager.storageserver.accounting.ReportFactory;
import org.gcube.contentmanager.storageserver.data.CubbyHole;
import org.gcube.contentmanager.storageserver.parse.utils.ValidationUtils;
import org.gcube.contentmanager.storageserver.store.FolderStatusOperationManager;
import org.gcube.contentmanager.storageserver.store.FolderStatusRecord;
import org.gcube.contentmanager.storageserver.store.FolderStatusObject;
import org.gcube.contentmanager.storageserver.store.MongoDB;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -114,18 +114,20 @@ public class FolderAccountingConsumer extends Thread{
}else{
// the record is a valid record for folder accounting
logger.info("this is a valid record for folder accounting. build folder record ");
FolderStatusRecord fsr=null;
FolderStatusObject fsr=null;
if(isNeedFSReport(lastOperation)){
try{
mongo=new MongoDB(server, user, password);
if(lastOperation.equalsIgnoreCase("COPY"))
owner=lastUser;
fsr=new FolderStatusRecord(filename, length, 1, lastAccess, from);
//extract folder from filename field
String folder=filename.substring(0, filename.lastIndexOf("/"));
fsr=new FolderStatusObject(folder, length, 1, lastAccess, from);
fsr=mongo.updateFolderVolume(fsr, lastOperation);
// if it is a Move operation it is need to update the original folder.
if(lastOperation.equalsIgnoreCase("MOVE")){
if((from != null)){
fsr=new FolderStatusRecord(from, length, 1, lastAccess, null);
fsr=new FolderStatusObject(from, length, 1, lastAccess, null);
fsr= mongo.updateFolderVolume(fsr, "DELETE");
}else{
logger.error("this is a move operation but the original path folder is missing. Maybe the storage-manager library is not update on the client. skip next");

6
src/main/java/org/gcube/contentmanager/storageserver/consumer/UserAccountingConsumer.java
View File

@ -10,7 +10,7 @@ import org.gcube.contentmanager.storageserver.accounting.ReportFactory;
import org.gcube.contentmanager.storageserver.data.CubbyHole;
import org.gcube.contentmanager.storageserver.parse.utils.ValidationUtils;
import org.gcube.contentmanager.storageserver.store.MongoDB;
import org.gcube.contentmanager.storageserver.store.StorageStatusRecord;
import org.gcube.contentmanager.storageserver.store.StorageStatusObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -127,13 +127,13 @@ public class UserAccountingConsumer extends Thread{
}
}
logger.debug(" operation accounted "+lastOperation);
StorageStatusRecord ssr=null;
StorageStatusObject ssr=null;
if(isNeedSSReport(lastOperation)){
try{
mongo=new MongoDB(server, user, password);
if(lastOperation.equalsIgnoreCase("COPY"))
owner=lastUser;
ssr=new StorageStatusRecord(owner, length, 1);
ssr=new StorageStatusObject(owner, length, 1);
ssr=mongo.updateUserVolume(ssr, lastOperation);
mongo.close();
}catch(Exception e){

6
src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusRecord.java → src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusObject.java
View File

@ -3,7 +3,7 @@ package org.gcube.contentmanager.storageserver.store;
import java.util.Date;
import com.mongodb.DBObject;
public class FolderStatusRecord {
public class FolderStatusObject {
private String folder;
@ -20,7 +20,7 @@ public class FolderStatusRecord {
private DBObject dbo;
public FolderStatusRecord(String folderPath, long volume, int count, String lastUpdate, String from){
public FolderStatusObject(String folderPath, long volume, int count, String lastUpdate, String from){
this.folder=folderPath;
this.volume=volume;
this.count=count;
@ -28,7 +28,7 @@ public class FolderStatusRecord {
this.originalFolder=from;
}
public FolderStatusRecord(String id, String folderPath, long volume, int count, String lastUpdate, String from, DBObject obj){
public FolderStatusObject(String id, String folderPath, long volume, int count, String lastUpdate, String from, DBObject obj){
this.id=id;
this.folder=folderPath;
this.volume=volume;

83
src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusOperationManager.java
View File

@ -19,47 +19,60 @@ public class FolderStatusOperationManager {
setFsCollection(fsCollection);
}
public void putFSRecord(String folder, long volume, int count, String lastUpdate){
public FolderStatusObject putFSRecord(String folder, long volume, int count, String lastUpdate){
BasicDBObject doc = new BasicDBObject("folder", folder)
.append("volume", volume)
.append("count", count)
.append("lastupdate", lastUpdate);
.append("lastUpdate", lastUpdate);
getFsCollection().insert(doc);
return new FolderStatusObject(folder, volume, count, lastUpdate, null);
}
public FolderStatusRecord updateFolder(FolderStatusRecord fsRecord, String lastOperation){
FolderStatusRecord oldFsr=getFSRecord(fsRecord.getFolder());
if(oldFsr != null){
int partialCount=oldFsr.getCount();
int count = countCalculation(fsRecord.getCount(), partialCount, lastOperation);
fsRecord.setCount(count);
long partialVolume=oldFsr.getVolume();
long volume = volumeCalculation(fsRecord.getVolume(), partialVolume, lastOperation);
fsRecord.setVolume(volume);
final BasicDBObject query = new BasicDBObject("folder", fsRecord.getFolder());
// Creating BasicDBObjectBuilder object without arguments
DBObject documentBuilder = BasicDBObjectBuilder.start()
.add("volume", volume).add("count", count).add("lastUpdate", fsRecord.getLastUpdate()).get();
// get the dbobject from builder and Inserting document
getFsCollection().update(query,new BasicDBObject("$set", documentBuilder), true, false);
// // if is a move operation then will be update also the source folder
// if(lastOperation.equalsIgnoreCase("MOVE")){
// FolderStatusRecord originalFolderRecord = getFSRecord(folder)
// }
public FolderStatusObject updateFolder(FolderStatusObject fsObject, String lastOperation){
if(fsObject!=null){
String currentFolder=fsObject.getFolder();
int currentCount=fsObject.getCount();
long currentVolume=fsObject.getVolume();
logger.info("update Folder: "+currentFolder+" with partialVolume "+fsObject.getVolume()+" and count: "+fsObject.getCount());
FolderStatusObject oldFsr=getFSRecord(currentFolder);
if(oldFsr != null){
logger.debug("this folder "+currentFolder+" is already present in the storage");
int partialCount=oldFsr.getCount();
int count = countCalculation(fsObject.getCount(), partialCount, lastOperation);
fsObject.setCount(count);
long partialVolume=oldFsr.getVolume();
long volume = volumeCalculation(fsObject.getVolume(), partialVolume, lastOperation);
fsObject.setVolume(volume);
final BasicDBObject query = new BasicDBObject("folder", fsObject.getFolder());
// Creating BasicDBObjectBuilder object without arguments
DBObject documentBuilder = BasicDBObjectBuilder.start()
.add("volume", volume).add("count", count).add("lastUpdate", fsObject.getLastUpdate()).get();
// get the dbobject from builder and Inserting document
getFsCollection().update(query,new BasicDBObject("$set", documentBuilder), true, false);
}else{
logger.debug("folder "+currentFolder+" not present yet");
putFSRecord(fsObject.getFolder(), currentVolume, currentCount, fsObject.getLastUpdate());
}
String parentDir=currentFolder.substring(0, currentFolder.lastIndexOf("/"));
logger.debug("recursive update from parent folder: "+parentDir);
FolderStatusObject parentObject=new FolderStatusObject(parentDir, currentVolume, 1, fsObject.getLastUpdate(), fsObject.getOriginalFolder());
if((parentDir !=null) && parentDir.contains("/"))
parentObject=updateFolder(parentObject, lastOperation);
return fsObject;
}else{
putFSRecord(fsRecord.getFolder(), fsRecord.getVolume(), fsRecord.getCount(), fsRecord.getLastUpdate());
logger.error("invalid invocation update method: record is null");
return null;
}
return fsRecord;
}
public FolderStatusRecord getFSRecord(String folder){
public FolderStatusObject getFSRecord(String folder){
BasicDBObject query = new BasicDBObject("folder", folder);
DBCursor cursor=getFsCollection().find(query);
DBObject obj=null;
@ -72,9 +85,9 @@ public class FolderStatusOperationManager {
cursor.close();
}
if(obj!=null){
String cons=null;
if(obj.containsField("consumer")) cons=(String) obj.get("consumer");
else logger.error("incomplete record found. consumer field is missing");
String currentFolder=null;
if(obj.containsField("folder")) currentFolder=(String) obj.get("folder");
else logger.error("incomplete record found. folder field is missing");
long vol =0;
if(obj.containsField("volume")) vol=(long) obj.get("volume");
else logger.error("incomplete record found. volume field is missing");
@ -86,9 +99,9 @@ public class FolderStatusOperationManager {
else logger.error("incomplete record found. lastUpdate field is missing");
String originalFolder=null;
if(obj.containsField("from")) originalFolder=(String) obj.get("from");
else logger.info(" originalFolder field is missing");
else logger.info(" originalFolder field is missing. This is correct if this isn't a move operation");
String id=(String)obj.get("id");
return new FolderStatusRecord(id, cons, vol, count, lastUpdate, originalFolder, obj);
return new FolderStatusObject(id, currentFolder, vol, count, lastUpdate, originalFolder, obj);
}else{
return null;
}
@ -106,7 +119,7 @@ public class FolderStatusOperationManager {
}
private int countCalculation(int currentCount, int partialCount, String operation) {
logger.info("accounting: operation "+operation+" total count "+partialCount+" current count"+currentCount);
logger.info(" operation "+operation+" partial count "+partialCount+" current count"+currentCount);
if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY"))
partialCount=partialCount+currentCount;
else if(operation.equalsIgnoreCase("DELETE"))

14
src/main/java/org/gcube/contentmanager/storageserver/store/MongoDB.java
View File

@ -66,26 +66,28 @@ public class MongoDB {
}
public StorageStatusRecord updateUserVolume(StorageStatusRecord ssRecord, String operation){
public StorageStatusObject updateUserVolume(StorageStatusObject ssRecord, String operation){
ssRecord= ssOperationManager.updateUser(ssRecord, operation);
close();
return ssRecord;
}
public FolderStatusRecord updateFolderVolume(FolderStatusRecord fsRecord, String operation){
public FolderStatusObject updateFolderVolume(FolderStatusObject fsRecord, String operation){
logger.debug("update folder recursively");
fsRecord= folderOperationManager.updateFolder(fsRecord, operation);
logger.debug("end update folder edge ");
close();
return fsRecord;
}
public StorageStatusRecord getSSRecord(String consumer){
StorageStatusRecord record=ssOperationManager.getSSRecord(consumer);
public StorageStatusObject getSSRecord(String consumer){
StorageStatusObject record=ssOperationManager.getSSRecord(consumer);
close();
return record;
}
public FolderStatusRecord getFSRecord(String folder){
FolderStatusRecord record=folderOperationManager.getFSRecord(folder);
public FolderStatusObject getFSRecord(String folder){
FolderStatusObject record=folderOperationManager.getFSRecord(folder);
close();
return record;
}

6
src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusRecord.java → src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusObject.java
View File

@ -2,7 +2,7 @@ package org.gcube.contentmanager.storageserver.store;
import com.mongodb.DBObject;
public class StorageStatusRecord {
public class StorageStatusObject {
private String consumer;
@ -14,13 +14,13 @@ public class StorageStatusRecord {
private DBObject dbo;
public StorageStatusRecord(String consumer, long volume, int count){
public StorageStatusObject(String consumer, long volume, int count){
this.consumer=consumer;
this.volume=volume;
this.count=count;
}
public StorageStatusRecord(String id, String consumer, long volume, int count, DBObject obj){
public StorageStatusObject(String id, String consumer, long volume, int count, DBObject obj){
this.id=id;
this.consumer=consumer;
this.volume=volume;

8
src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusOperationManager.java
View File

@ -25,8 +25,8 @@ public class StorageStatusOperationManager {
getSsCollection().insert(doc);
}
public StorageStatusRecord updateUser(StorageStatusRecord ssRecord, String lastOperation){
StorageStatusRecord oldSsr=getSSRecord(ssRecord.getConsumer());
public StorageStatusObject updateUser(StorageStatusObject ssRecord, String lastOperation){
StorageStatusObject oldSsr=getSSRecord(ssRecord.getConsumer());
if(oldSsr != null){
int partialCount=oldSsr.getCount();
@ -52,7 +52,7 @@ public class StorageStatusOperationManager {
public StorageStatusRecord getSSRecord(String consumer){
public StorageStatusObject getSSRecord(String consumer){
BasicDBObject query = new BasicDBObject("consumer", consumer);
DBCursor cursor=getSsCollection().find(query);
DBObject obj=null;
@ -75,7 +75,7 @@ public class StorageStatusOperationManager {
if(obj.containsField("count")) count=(int) obj.get("count");
else logger.error("incomplete record found. count field is missing");
String id=(String)obj.get("id");
return new StorageStatusRecord(id, cons, vol, count, obj);
return new StorageStatusObject(id, cons, vol, count, obj);
}else{
return null;
}

2
src/test/java/org/gcube/contentmanager/storageserver/store/MongoDBTest.java
View File

@ -15,7 +15,7 @@ public class MongoDBTest {
// @Test
public void update(){
StorageStatusRecord ssr=new StorageStatusRecord("test.consumer", 100, 1);
StorageStatusObject ssr=new StorageStatusObject("test.consumer", 100, 1);
mongo.updateUserVolume(ssr, "UPLOAD");
}

23
src/test/java/org/gcube/contentmanager/storageserver/test/StringFolderTest.java Normal file
View File

@ -0,0 +1,23 @@
package org.gcube.contentmanager.storageserver.test;
import static org.junit.Assert.*;
import org.junit.Test;
public class StringFolderTest {
String currentFolder="a/b/c/d";
@Test
public void test() {
System.out.println("folder start: "+currentFolder);
while(currentFolder!= null && currentFolder.contains("/")){
currentFolder=currentFolder.substring(0, currentFolder.lastIndexOf("/"));
System.out.println("dir parent: "+currentFolder);
}
System.out.println("folder end: "+currentFolder);
}
}