gCubeSystem
/
storage-manager-trigger
16
0
Fork 0
Code Issues Releases Wiki Activity

bugfix on thread manage 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/storage-manager-trigger@100927 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
roberto.cirillo 2014-10-23 16:08:31 +00:00
parent 6a578aa947
commit 1cac8ba8e6
6 changed files with 47 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/main/java/org/gcube/contentmanager/storageserver/consumer/FolderAccountingConsumer.java
View File

@ -77,7 +77,7 @@ public class FolderAccountingConsumer extends Thread{
MongoDB mongo=null; MongoDB mongo=null;
try{ try{
x=c.get(); x=c.get();
logger.debug("Consumer #" + this.number + " got: " + x); logger.debug("[FolderACCOUNTING] Consumer #" + this.number + " got: " + x);
op = (String) x.get("op"); op = (String) x.get("op");
// retrieve object fields // retrieve object fields
DBObject obj=(DBObject)x.get("o"); DBObject obj=(DBObject)x.get("o");
@ -85,7 +85,7 @@ public class FolderAccountingConsumer extends Thread{
// set object dimension // set object dimension
long length=-1; long length=-1;
if(obj.get("length")!=null) length=(long)obj.get("length"); if(obj.get("length")!=null) length=(long)obj.get("length");
logger.debug("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+"\n\t cretionTime: "+creationTime+ " lastOperation "+lastOperation+" lastUser: "+lastUser+" lastAccess: "+lastAccess); logger.info("[FolderACCOUNTING][recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+"\n\t cretionTime: "+creationTime+ " lastOperation "+lastOperation+" lastUser: "+lastUser+" lastAccess: "+lastAccess);
if(((length >0) && (((filename!=null) && (filename.contains("/"))) || (linkCount > 0)))){ if(((length >0) && (((filename!=null) && (filename.contains("/"))) || (linkCount > 0)))){
//convert from byte to kb //convert from byte to kb
length=length/1024; length=length/1024;
@ -98,22 +98,23 @@ public class FolderAccountingConsumer extends Thread{
String pathString=(String)obj.get("onScope"); String pathString=(String)obj.get("onScope");
scope=retrieveScopeFromRemoteFilePath(pathString); scope=retrieveScopeFromRemoteFilePath(pathString);
} }
logger.info("update folders: "+filename); logger.info("[FolderACCOUNTING] update edge: "+filename);
logger.info("\n[FolderAccountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp); // logger.info("\n[FolderACCOUNTING] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp);
boolean validScope=ValidationUtils.validationScope(scope); boolean validScope=ValidationUtils.validationScope(scope);
if(validScope){ if(validScope){
if(delete!=null){ if(delete!=null){
lastOperation="DELETE"; lastOperation="DELETE";
}else if ((lastOperation != null) && (op != null) && (lastOperation.equalsIgnoreCase("LINK")) && (op.equalsIgnoreCase("u"))){ }else if ((lastOperation != null) && (op != null) && (lastOperation.equalsIgnoreCase("LINK")) && (op.equalsIgnoreCase("u"))){
// it is an update on a link object this operation doesn't be accounted // it is an update on a link object this operation doesn't be accounted
logger.info("update on link object is not accounted. Skip next "); logger.info("[FolderACCOUNTING] update on link object is not accounted. Skip next ");
continue; continue;
}else if((lastOperation==null) || (lastUser==null)){ }else if((lastOperation==null) || (lastUser==null)){
logger.warn("lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next "); logger.warn("[FolderACCOUNTING] lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next ");
continue; continue;
}else{ }//else{
// the record is a valid record for folder accounting // the record is a valid record for folder accounting
logger.info("this is a valid record for folder accounting. build folder record "); logger.info("[FolderACCOUNTING] this is a valid record for folder accounting. build folder record ");
print(scope, length);
FolderStatusObject fsr=null; FolderStatusObject fsr=null;
if(isNeedFSReport(lastOperation)){ if(isNeedFSReport(lastOperation)){
try{ try{
@ -135,10 +136,12 @@ public class FolderAccountingConsumer extends Thread{
} }
mongo.close(); mongo.close();
}catch(Exception e){ }catch(Exception e){
logger.error("Problem in updating storage status record: "+e.getMessage()); logger.error("Problem in updating folder status record: "+e.getMessage());
} }
} }
} // }
}else{
logger.error("invalid scope found"+scope+" skip next record" );
} }
}else{ }else{
logger.info("operation "+lastOperation+" is not accounted"); logger.info("operation "+lastOperation+" is not accounted");
@ -155,6 +158,8 @@ public class FolderAccountingConsumer extends Thread{
} }
private boolean isNeedSSReport(String lastOperation) { private boolean isNeedSSReport(String lastOperation) {
if(lastOperation.equalsIgnoreCase("UPLOAD") || lastOperation.equalsIgnoreCase("COPY") || lastOperation.equalsIgnoreCase("DELETE")) if(lastOperation.equalsIgnoreCase("UPLOAD") || lastOperation.equalsIgnoreCase("COPY") || lastOperation.equalsIgnoreCase("DELETE"))
return true; return true;
@ -195,31 +200,20 @@ public class FolderAccountingConsumer extends Thread{
// ObjectId objectId=(ObjectId)obj.get("_id"); // ObjectId objectId=(ObjectId)obj.get("_id");
// id = objectId.toString(); // id = objectId.toString();
} }
private void report(String filename, String owner, private void print(String scope, long length) {
String creationTime, long length, logger.info(" Folder accounting properties: " +
String scope, String operation, String callerIP, String lastAccess, String lastUser, String totVolume, String totCount) { "\n\t owner: "+owner+
// ACCOUNTING CALL TYPE: STORAGE USAGE "\n\t scope: "+scope+
RawUsageRecord sur=report.setGenericProperties("storage-usage",lastUser, scope, creationTime, lastAccess, owner); "\n\t type folder-status"+
sur=report.setSpecificProperties(sur, operation, length+"", filename, callerIP, "STORAGE", "1"); "\n\t file "+filename+
logger.info("[accounting call] type: storage usage "); "\n\t size "+length+
report.printRecord(sur); "\n\t caller "+callerIp+
report.send(sur); "\n\t consumer "+lastUser+
// ACCOUNTING CALL TYPE: STORAGE STATUS "\n\t lastAccess "+lastAccess+
logger.debug("set properties: totVolume: "+totVolume+" totCount "+totCount); "\n\t operation "+lastOperation);
if((totVolume!=null) && (totCount!=null)){
RawUsageRecord ssr=report.setGenericProperties("storage-status",lastUser, scope, creationTime, lastAccess, owner);
ssr=report.setSpecificProperties(ssr, operation, totVolume, filename, callerIP, "STORAGE", totCount);
logger.info("[accounting call] type: storage status ");
report.printRecord(ssr);
report.send(ssr);
}else{
logger.info("StorageStatus record not send for operation "+lastOperation);
}
} }
private String retrieveScopeFromRemoteFilePath(String filename) { private String retrieveScopeFromRemoteFilePath(String filename) {
String[] split=filename.split("/"); String[] split=filename.split("/");

2
src/main/java/org/gcube/contentmanager/storageserver/consumer/UserAccountingConsumer.java
View File

@ -252,7 +252,7 @@ public class UserAccountingConsumer extends Thread{
scope=scope+"/"+split[i]; scope=scope+"/"+split[i];
i++; i++;
} }
logger.info("retieved scope: "+scope); logger.debug("retieved scope: "+scope);
return scope; return scope;
}else logger.error("Scope bad format: scope not retrieved from string: "+filename); }else logger.error("Scope bad format: scope not retrieved from string: "+filename);
return null; return null;

12
src/main/java/org/gcube/contentmanager/storageserver/data/CubbyHole.java
View File

@ -32,12 +32,12 @@ public class CubbyHole {
} }
public synchronized void put(DBObject value) { public synchronized void put(DBObject value) {
// while (available == true) { while (available == true) {
// try { try {
// wait(); wait();
// } catch (InterruptedException e) { } catch (InterruptedException e) {
// } }
// } }
logger.debug("put element to queue: "+value); logger.debug("put element to queue: "+value);
requestQueue.addElement(value); requestQueue.addElement(value);
available = true; available = true;

4
src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java
View File

@ -57,9 +57,9 @@ public class Startup {
ssConsumer=new UserAccountingConsumer(server, args[1], args[2], c1, 1, dtsHosts); ssConsumer=new UserAccountingConsumer(server, args[1], args[2], c1, 1, dtsHosts);
FolderAccountingConsumer fsConsumer=null; FolderAccountingConsumer fsConsumer=null;
if(args.length == 3) if(args.length == 3)
fsConsumer=new FolderAccountingConsumer(server, c1, 1); fsConsumer=new FolderAccountingConsumer(server, c2, 1);
else else
fsConsumer=new FolderAccountingConsumer(server, args[1], args[2], c1, 1); fsConsumer=new FolderAccountingConsumer(server, args[1], args[2], c2, 1);
producer.start(); producer.start();
ssConsumer.start(); ssConsumer.start();
fsConsumer.start(); fsConsumer.start();

12
src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusOperationManager.java
View File

@ -34,7 +34,7 @@ public class FolderStatusOperationManager {
String currentFolder=fsObject.getFolder(); String currentFolder=fsObject.getFolder();
int currentCount=fsObject.getCount(); int currentCount=fsObject.getCount();
long currentVolume=fsObject.getVolume(); long currentVolume=fsObject.getVolume();
logger.info("update Folder: "+currentFolder+" with partialVolume "+fsObject.getVolume()+" and count: "+fsObject.getCount()); logger.debug("update Folder: "+currentFolder+" with partialVolume "+fsObject.getVolume()+" and count: "+fsObject.getCount());
FolderStatusObject oldFsr=getFSRecord(currentFolder); FolderStatusObject oldFsr=getFSRecord(currentFolder);
if(oldFsr != null){ if(oldFsr != null){
logger.debug("this folder "+currentFolder+" is already present in the storage"); logger.debug("this folder "+currentFolder+" is already present in the storage");
@ -99,7 +99,7 @@ public class FolderStatusOperationManager {
else logger.error("incomplete record found. lastUpdate field is missing"); else logger.error("incomplete record found. lastUpdate field is missing");
String originalFolder=null; String originalFolder=null;
if(obj.containsField("from")) originalFolder=(String) obj.get("from"); if(obj.containsField("from")) originalFolder=(String) obj.get("from");
else logger.info(" originalFolder field is missing. This is correct if this isn't a move operation"); else logger.debug(" originalFolder field is missing. This is correct if this isn't a move operation");
String id=(String)obj.get("id"); String id=(String)obj.get("id");
return new FolderStatusObject(id, currentFolder, vol, count, lastUpdate, originalFolder, obj); return new FolderStatusObject(id, currentFolder, vol, count, lastUpdate, originalFolder, obj);
}else{ }else{
@ -108,23 +108,23 @@ public class FolderStatusOperationManager {
} }
private long volumeCalculation(long currentVolume, long partialVolume, String operation) { private long volumeCalculation(long currentVolume, long partialVolume, String operation) {
logger.info("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume); logger.debug("folder accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume);
if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){ if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){
partialVolume=partialVolume+currentVolume; partialVolume=partialVolume+currentVolume;
}else if(operation.equalsIgnoreCase("DELETE")){ }else if(operation.equalsIgnoreCase("DELETE")){
partialVolume=partialVolume-currentVolume; partialVolume=partialVolume-currentVolume;
} }
logger.info("new volume "+partialVolume); logger.debug("new volume "+partialVolume);
return partialVolume; return partialVolume;
} }
private int countCalculation(int currentCount, int partialCount, String operation) { private int countCalculation(int currentCount, int partialCount, String operation) {
logger.info(" operation "+operation+" partial count "+partialCount+" current count"+currentCount); logger.debug("folder accounting operation "+operation+" partial count "+partialCount+" current count"+currentCount);
if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY")) if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY"))
partialCount=partialCount+currentCount; partialCount=partialCount+currentCount;
else if(operation.equalsIgnoreCase("DELETE")) else if(operation.equalsIgnoreCase("DELETE"))
partialCount=partialCount-currentCount; partialCount=partialCount-currentCount;
logger.info("new count: "+partialCount); logger.debug("new count: "+partialCount);
return partialCount; return partialCount;
} }

8
src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusOperationManager.java
View File

@ -82,23 +82,23 @@ public class StorageStatusOperationManager {
} }
private long volumeCalculation(long currentVolume, long partialVolume, String operation) { private long volumeCalculation(long currentVolume, long partialVolume, String operation) {
logger.info("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume); logger.debug("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume);
if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){ if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){
partialVolume=partialVolume+currentVolume; partialVolume=partialVolume+currentVolume;
}else if(operation.equalsIgnoreCase("DELETE")){ }else if(operation.equalsIgnoreCase("DELETE")){
partialVolume=partialVolume-currentVolume; partialVolume=partialVolume-currentVolume;
} }
logger.info("new volume "+partialVolume); logger.debug("new volume "+partialVolume);
return partialVolume; return partialVolume;
} }
private int countCalculation(int currentCount, int partialCount, String operation) { private int countCalculation(int currentCount, int partialCount, String operation) {
logger.info("accounting: operation "+operation+" total count "+partialCount+" current count"+currentCount); logger.debug("accounting: operation "+operation+" total count "+partialCount+" current count"+currentCount);
if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY")) if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY"))
partialCount=partialCount+currentCount; partialCount=partialCount+currentCount;
else if(operation.equalsIgnoreCase("DELETE")) else if(operation.equalsIgnoreCase("DELETE"))
partialCount=partialCount-currentCount; partialCount=partialCount-currentCount;
logger.info("new count: "+partialCount); logger.debug("new count: "+partialCount);
return partialCount; return partialCount;
} }