diff --git a/src/main/java/org/gcube/contentmanager/storageserver/consumer/FolderAccountingConsumer.java b/src/main/java/org/gcube/contentmanager/storageserver/consumer/FolderAccountingConsumer.java index 9cdc0ec..4d19eeb 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/consumer/FolderAccountingConsumer.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/consumer/FolderAccountingConsumer.java @@ -77,7 +77,7 @@ public class FolderAccountingConsumer extends Thread{ MongoDB mongo=null; try{ x=c.get(); - logger.debug("Consumer #" + this.number + " got: " + x); + logger.debug("[FolderACCOUNTING] Consumer #" + this.number + " got: " + x); op = (String) x.get("op"); // retrieve object fields DBObject obj=(DBObject)x.get("o"); @@ -85,7 +85,7 @@ public class FolderAccountingConsumer extends Thread{ // set object dimension long length=-1; if(obj.get("length")!=null) length=(long)obj.get("length"); - logger.debug("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+"\n\t cretionTime: "+creationTime+ " lastOperation "+lastOperation+" lastUser: "+lastUser+" lastAccess: "+lastAccess); + logger.info("[FolderACCOUNTING][recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+"\n\t cretionTime: "+creationTime+ " lastOperation "+lastOperation+" lastUser: "+lastUser+" lastAccess: "+lastAccess); if(((length >0) && (((filename!=null) && (filename.contains("/"))) || (linkCount > 0)))){ //convert from byte to kb length=length/1024; @@ -98,22 +98,23 @@ public class FolderAccountingConsumer extends Thread{ String pathString=(String)obj.get("onScope"); scope=retrieveScopeFromRemoteFilePath(pathString); } - logger.info("update folders: "+filename); - logger.info("\n[FolderAccountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp); + logger.info("[FolderACCOUNTING] update edge: "+filename); +// logger.info("\n[FolderACCOUNTING] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp); boolean validScope=ValidationUtils.validationScope(scope); if(validScope){ if(delete!=null){ lastOperation="DELETE"; }else if ((lastOperation != null) && (op != null) && (lastOperation.equalsIgnoreCase("LINK")) && (op.equalsIgnoreCase("u"))){ // it is an update on a link object this operation doesn't be accounted - logger.info("update on link object is not accounted. Skip next "); + logger.info("[FolderACCOUNTING] update on link object is not accounted. Skip next "); continue; }else if((lastOperation==null) || (lastUser==null)){ - logger.warn("lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next "); + logger.warn("[FolderACCOUNTING] lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next "); continue; - }else{ + }//else{ // the record is a valid record for folder accounting - logger.info("this is a valid record for folder accounting. build folder record "); + logger.info("[FolderACCOUNTING] this is a valid record for folder accounting. build folder record "); + print(scope, length); FolderStatusObject fsr=null; if(isNeedFSReport(lastOperation)){ try{ @@ -135,10 +136,12 @@ public class FolderAccountingConsumer extends Thread{ } mongo.close(); }catch(Exception e){ - logger.error("Problem in updating storage status record: "+e.getMessage()); + logger.error("Problem in updating folder status record: "+e.getMessage()); } } - } +// } + }else{ + logger.error("invalid scope found"+scope+" skip next record" ); } }else{ logger.info("operation "+lastOperation+" is not accounted"); @@ -155,6 +158,8 @@ public class FolderAccountingConsumer extends Thread{ } + + private boolean isNeedSSReport(String lastOperation) { if(lastOperation.equalsIgnoreCase("UPLOAD") || lastOperation.equalsIgnoreCase("COPY") || lastOperation.equalsIgnoreCase("DELETE")) return true; @@ -195,31 +200,20 @@ public class FolderAccountingConsumer extends Thread{ // ObjectId objectId=(ObjectId)obj.get("_id"); // id = objectId.toString(); } - - private void report(String filename, String owner, - String creationTime, long length, - String scope, String operation, String callerIP, String lastAccess, String lastUser, String totVolume, String totCount) { -// ACCOUNTING CALL TYPE: STORAGE USAGE - RawUsageRecord sur=report.setGenericProperties("storage-usage",lastUser, scope, creationTime, lastAccess, owner); - sur=report.setSpecificProperties(sur, operation, length+"", filename, callerIP, "STORAGE", "1"); - logger.info("[accounting call] type: storage usage "); - report.printRecord(sur); - report.send(sur); -// ACCOUNTING CALL TYPE: STORAGE STATUS - logger.debug("set properties: totVolume: "+totVolume+" totCount "+totCount); - if((totVolume!=null) && (totCount!=null)){ - RawUsageRecord ssr=report.setGenericProperties("storage-status",lastUser, scope, creationTime, lastAccess, owner); - ssr=report.setSpecificProperties(ssr, operation, totVolume, filename, callerIP, "STORAGE", totCount); - logger.info("[accounting call] type: storage status "); - report.printRecord(ssr); - report.send(ssr); - }else{ - logger.info("StorageStatus record not send for operation "+lastOperation); - } + + private void print(String scope, long length) { + logger.info(" Folder accounting properties: " + + "\n\t owner: "+owner+ + "\n\t scope: "+scope+ + "\n\t type folder-status"+ + "\n\t file "+filename+ + "\n\t size "+length+ + "\n\t caller "+callerIp+ + "\n\t consumer "+lastUser+ + "\n\t lastAccess "+lastAccess+ + "\n\t operation "+lastOperation); + } - - - private String retrieveScopeFromRemoteFilePath(String filename) { String[] split=filename.split("/"); diff --git a/src/main/java/org/gcube/contentmanager/storageserver/consumer/UserAccountingConsumer.java b/src/main/java/org/gcube/contentmanager/storageserver/consumer/UserAccountingConsumer.java index a1876c3..5bb902b 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/consumer/UserAccountingConsumer.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/consumer/UserAccountingConsumer.java @@ -252,7 +252,7 @@ public class UserAccountingConsumer extends Thread{ scope=scope+"/"+split[i]; i++; } - logger.info("retieved scope: "+scope); + logger.debug("retieved scope: "+scope); return scope; }else logger.error("Scope bad format: scope not retrieved from string: "+filename); return null; diff --git a/src/main/java/org/gcube/contentmanager/storageserver/data/CubbyHole.java b/src/main/java/org/gcube/contentmanager/storageserver/data/CubbyHole.java index fca2aa8..c6643be 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/data/CubbyHole.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/data/CubbyHole.java @@ -32,12 +32,12 @@ public class CubbyHole { } public synchronized void put(DBObject value) { -// while (available == true) { -// try { -// wait(); -// } catch (InterruptedException e) { -// } -// } + while (available == true) { + try { + wait(); + } catch (InterruptedException e) { + } + } logger.debug("put element to queue: "+value); requestQueue.addElement(value); available = true; diff --git a/src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java b/src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java index 3cb4ccf..3da769d 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java @@ -57,9 +57,9 @@ public class Startup { ssConsumer=new UserAccountingConsumer(server, args[1], args[2], c1, 1, dtsHosts); FolderAccountingConsumer fsConsumer=null; if(args.length == 3) - fsConsumer=new FolderAccountingConsumer(server, c1, 1); + fsConsumer=new FolderAccountingConsumer(server, c2, 1); else - fsConsumer=new FolderAccountingConsumer(server, args[1], args[2], c1, 1); + fsConsumer=new FolderAccountingConsumer(server, args[1], args[2], c2, 1); producer.start(); ssConsumer.start(); fsConsumer.start(); diff --git a/src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusOperationManager.java b/src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusOperationManager.java index 9977684..f58d23f 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusOperationManager.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/store/FolderStatusOperationManager.java @@ -34,7 +34,7 @@ public class FolderStatusOperationManager { String currentFolder=fsObject.getFolder(); int currentCount=fsObject.getCount(); long currentVolume=fsObject.getVolume(); - logger.info("update Folder: "+currentFolder+" with partialVolume "+fsObject.getVolume()+" and count: "+fsObject.getCount()); + logger.debug("update Folder: "+currentFolder+" with partialVolume "+fsObject.getVolume()+" and count: "+fsObject.getCount()); FolderStatusObject oldFsr=getFSRecord(currentFolder); if(oldFsr != null){ logger.debug("this folder "+currentFolder+" is already present in the storage"); @@ -99,7 +99,7 @@ public class FolderStatusOperationManager { else logger.error("incomplete record found. lastUpdate field is missing"); String originalFolder=null; if(obj.containsField("from")) originalFolder=(String) obj.get("from"); - else logger.info(" originalFolder field is missing. This is correct if this isn't a move operation"); + else logger.debug(" originalFolder field is missing. This is correct if this isn't a move operation"); String id=(String)obj.get("id"); return new FolderStatusObject(id, currentFolder, vol, count, lastUpdate, originalFolder, obj); }else{ @@ -108,23 +108,23 @@ public class FolderStatusOperationManager { } private long volumeCalculation(long currentVolume, long partialVolume, String operation) { - logger.info("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume); + logger.debug("folder accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume); if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){ partialVolume=partialVolume+currentVolume; }else if(operation.equalsIgnoreCase("DELETE")){ partialVolume=partialVolume-currentVolume; } - logger.info("new volume "+partialVolume); + logger.debug("new volume "+partialVolume); return partialVolume; } private int countCalculation(int currentCount, int partialCount, String operation) { - logger.info(" operation "+operation+" partial count "+partialCount+" current count"+currentCount); + logger.debug("folder accounting operation "+operation+" partial count "+partialCount+" current count"+currentCount); if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY")) partialCount=partialCount+currentCount; else if(operation.equalsIgnoreCase("DELETE")) partialCount=partialCount-currentCount; - logger.info("new count: "+partialCount); + logger.debug("new count: "+partialCount); return partialCount; } diff --git a/src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusOperationManager.java b/src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusOperationManager.java index 17e1232..fd75d61 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusOperationManager.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/store/StorageStatusOperationManager.java @@ -82,23 +82,23 @@ public class StorageStatusOperationManager { } private long volumeCalculation(long currentVolume, long partialVolume, String operation) { - logger.info("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume); + logger.debug("accounting: operation "+operation+" total Volume "+partialVolume+" current volume "+currentVolume); if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){ partialVolume=partialVolume+currentVolume; }else if(operation.equalsIgnoreCase("DELETE")){ partialVolume=partialVolume-currentVolume; } - logger.info("new volume "+partialVolume); + logger.debug("new volume "+partialVolume); return partialVolume; } private int countCalculation(int currentCount, int partialCount, String operation) { - logger.info("accounting: operation "+operation+" total count "+partialCount+" current count"+currentCount); + logger.debug("accounting: operation "+operation+" total count "+partialCount+" current count"+currentCount); if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY")) partialCount=partialCount+currentCount; else if(operation.equalsIgnoreCase("DELETE")) partialCount=partialCount-currentCount; - logger.info("new count: "+partialCount); + logger.debug("new count: "+partialCount); return partialCount; }