gCubeSystem
/
storage-manager-trigger
16
0
Fork 0
Code Issues Releases Wiki Activity

bugfix on thread manage 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/storage-manager-trigger@100927 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
roberto.cirillo 2014-10-23 16:08:31 +00:00
parent 6a578aa947
commit 1cac8ba8e6
6 changed files with 47 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/main/java/org/gcube/contentmanager/storageserver/consumer/FolderAccountingConsumer.java
View File

@ -77,7 +77,7 @@ public class FolderAccountingConsumer extends Thread{
MongoDB mongo=null;
try{
x=c.get();
logger.debug("Consumer #" + this.number + " got: " + x);
logger.debug("[FolderACCOUNTING] Consumer #" + this.number + " got: " + x);
op = (String) x.get("op");
// retrieve object fields
DBObject obj=(DBObject)x.get("o");
@ -85,7 +85,7 @@ public class FolderAccountingConsumer extends Thread{
// set object dimension
long length=-1;
if(obj.get("length")!=null) length=(long)obj.get("length");
logger.debug("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+"\n\t cretionTime: "+creationTime+ " lastOperation "+lastOperation+" lastUser: "+lastUser+" lastAccess: "+lastAccess);
logger.info("[FolderACCOUNTING][recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+"\n\t cretionTime: "+creationTime+ " lastOperation "+lastOperation+" lastUser: "+lastUser+" lastAccess: "+lastAccess);
if(((length >0) && (((filename!=null) && (filename.contains("/"))) || (linkCount > 0)))){
//convert from byte to kb
length=length/1024;
@ -98,22 +98,23 @@ public class FolderAccountingConsumer extends Thread{
String pathString=(String)obj.get("onScope");
scope=retrieveScopeFromRemoteFilePath(pathString);
}
logger.info("update folders: "+filename);
logger.info("\n[FolderAccountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp);
logger.info("[FolderACCOUNTING] update edge: "+filename);
// logger.info("\n[FolderACCOUNTING] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp);
boolean validScope=ValidationUtils.validationScope(scope);
if(validScope){
if(delete!=null){
lastOperation="DELETE";
}else if ((lastOperation != null) && (op != null) && (lastOperation.equalsIgnoreCase("LINK")) && (op.equalsIgnoreCase("u"))){
// it is an update on a link object this operation doesn't be accounted
logger.info("update on link object is not accounted. Skip next ");
logger.info("[FolderACCOUNTING] update on link object is not accounted. Skip next ");
continue;
}else if((lastOperation==null) || (lastUser==null)){
logger.warn("lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next ");
logger.warn("[FolderACCOUNTING] lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next ");
continue;
}else{
}//else{
// the record is a valid record for folder accounting
logger.info("this is a valid record for folder accounting. build folder record ");
logger.info("[FolderACCOUNTING] this is a valid record for folder accounting. build folder record ");
print(scope, length);
FolderStatusObject fsr=null;
if(isNeedFSReport(lastOperation)){
try{
@ -135,10 +136,12 @@ public class FolderAccountingConsumer extends Thread{
}
mongo.close();
}catch(Exception e){
logger.error("Problem in updating storage status record: "+e.getMessage());
logger.error("Problem in updating folder status record: "+e.getMessage());
}
}
}
// }
}else{
logger.error("invalid scope found"+scope+" skip next record" );
}
}else{
logger.info("operation "+lastOperation+" is not accounted");
@ -155,6 +158,8 @@ public class FolderAccountingConsumer extends Thread{
}
private boolean isNeedSSReport(String lastOperation) {
if(lastOperation.equalsIgnoreCase("UPLOAD") || lastOperation.equalsIgnoreCase("COPY") || lastOperation.equalsIgnoreCase("DELETE"))
return true;
@ -195,31 +200,20 @@ public class FolderAccountingConsumer extends Thread{
// ObjectId objectId=(ObjectId)obj.get("_id");
// id = objectId.toString();
}
private void report(String filename, String owner,
String creationTime, long length,
String scope, String operation, String callerIP, String lastAccess, String lastUser, String totVolume, String totCount) {
// ACCOUNTING CALL TYPE: STORAGE USAGE
RawUsageRecord sur=report.setGenericProperties("storage-usage",lastUser, scope, creationTime, lastAccess, owner);
sur=report.setSpecificProperties(sur, operation, length+"", filename, callerIP, "STORAGE", "1");
logger.info("[accounting call] type: storage usage ");
report.printRecord(sur);
report.send(sur);
// ACCOUNTING CALL TYPE: STORAGE STATUS
logger.debug("set properties: totVolume: "+totVolume+" totCount "+totCount);
if((totVolume!=null) && (totCount!=null)){
RawUsageRecord ssr=report.setGenericProperties("storage-status",lastUser, scope, creationTime, lastAccess, owner);
ssr=report.setSpecificProperties(ssr, operation, totVolume, filename, callerIP, "STORAGE", totCount);
logger.info("[accounting call] type: storage status ");
report.printRecord(ssr);
report.send(ssr);
}else{
logger.info("StorageStatus record not send for operation "+lastOperation);
}
private void print(String scope, long length) {
logger.info(" Folder accounting properties: " +
"\n\t owner: "+owner+
"\n\t scope: "+scope+
"\n\t type folder-status"+
"\n\t file "+filename+
"\n\t size "+length+
"\n\t caller "+callerIp+
"\n\t consumer "+lastUser+
"\n\t lastAccess "+lastAccess+
"\n\t operation "+lastOperation);
}
private String retrieveScopeFromRemoteFilePath(String filename) {
String[] split=filename.split("/");

2
src/main/java/org/gcube/contentmanager/storageserver/consumer/UserAccountingConsumer.java
View File

@ -252,7 +252,7 @@ public class UserAccountingConsumer extends Thread{
scope=scope+"/"+split[i];
i++;
}
logger.info("retieved scope: "+scope);
logger.debug("retieved scope: "+scope);
return scope;
}else logger.error("Scope bad format: scope not retrieved from string: "+filename);
return null;

12
src/main/java/org/gcube/contentmanager/storageserver/data/CubbyHole.java
View File

@ -32,12 +32,12 @@ public class CubbyHole {
}
public synchronized void put(DBObject value) {
// while (available == true) {
// try {
// wait();
// } catch (InterruptedException e) {
// }
// }
while (available == true) {
try {
wait();
} catch (InterruptedException e) {
}
}
logger.debug("put element to queue: "+value);
requestQueue.addElement(value);
available = true;

4
src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java
View File

@ -57,9 +57,9 @@ public class Startup {
ssConsumer=new UserAccountingConsumer(server, args[1], args[2], c1, 1, dtsHosts);
FolderAccountingConsumer fsConsumer=null;
if(args.length == 3)
fsConsumer=new FolderAccountingConsumer(server, c1, 1);
fsConsumer=new FolderAccountingConsumer(server, c2, 1);
else
fsConsumer=new FolderAccountingConsumer(server, args[1], args[2], c1, 1);
fsConsumer=new FolderAccountingConsumer(server, args[1], args[2], c2, 1);
producer.start();
ssConsumer.start();
fsConsumer.start();

12
src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusOperationManager.java
View File

@ -34,7 +34,7 @@ public class FolderStatusOperationManager {
String currentFolder=fsObject.getFolder();
int currentCount=fsObject.getCount();
long currentVolume=fsObject.getVolume();
logger.info("update Folder: "+currentFolder+" with partialVolume "+fsObject.getVolume()+" and count: "+fsObject.getCount());
logger.debug("update Folder: "+currentFolder+" with partialVolume "+fsObject.getVolume()+" and count: "+fsObject.getCount());
FolderStatusObject oldFsr=getFSRecord(currentFolder);
if(oldFsr != null){
logger.debug("this folder "+currentFolder+" is already present in the storage");
@ -99,7 +99,7 @@ public class FolderStatusOperationManager {
else logger.error("incomplete record found. lastUpdate field is missing");
String originalFolder=null;
if(obj.containsField("from")) originalFolder=(String) obj.get("from");
else logger.info(" originalFolder field is missing. This is correct if this isn't a move operation");
else logger.debug(" originalFolder field is missing. This is correct if this isn't a move operation");
String id=(String)obj.get("id");
return new FolderStatusObject(id, currentFolder, vol, count, lastUpdate, originalFolder, obj);
}else{
@ -108,23 +108,23 @@ public class FolderStatusOperationManager {
}
private long volumeCalculation(long currentVolume, long partialVolume, String operation) {
logger.info("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume);
logger.debug("folder accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume);
if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){
partialVolume=partialVolume+currentVolume;
}else if(operation.equalsIgnoreCase("DELETE")){
partialVolume=partialVolume-currentVolume;
}
logger.info("new volume "+partialVolume);
logger.debug("new volume "+partialVolume);
return partialVolume;
}
private int countCalculation(int currentCount, int partialCount, String operation) {
logger.info(" operation "+operation+" partial count "+partialCount+" current count"+currentCount);
logger.debug("folder accounting operation "+operation+" partial count "+partialCount+" current count"+currentCount);
if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY"))
partialCount=partialCount+currentCount;
else if(operation.equalsIgnoreCase("DELETE"))
partialCount=partialCount-currentCount;
logger.info("new count: "+partialCount);
logger.debug("new count: "+partialCount);
return partialCount;
}

8
src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusOperationManager.java
View File

@ -82,23 +82,23 @@ public class StorageStatusOperationManager {
}
private long volumeCalculation(long currentVolume, long partialVolume, String operation) {
logger.info("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume);
logger.debug("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume);
if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){
partialVolume=partialVolume+currentVolume;
}else if(operation.equalsIgnoreCase("DELETE")){
partialVolume=partialVolume-currentVolume;
}
logger.info("new volume "+partialVolume);
logger.debug("new volume "+partialVolume);
return partialVolume;
}
private int countCalculation(int currentCount, int partialCount, String operation) {
logger.info("accounting: operation "+operation+" total count "+partialCount+" current count"+currentCount);
logger.debug("accounting: operation "+operation+" total count "+partialCount+" current count"+currentCount);
if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY"))
partialCount=partialCount+currentCount;
else if(operation.equalsIgnoreCase("DELETE"))
partialCount=partialCount-currentCount;
logger.info("new count: "+partialCount);
logger.debug("new count: "+partialCount);
return partialCount;
}