diff --git a/distro/CHANGELOG b/distro/CHANGELOG index 80dacc0..c7e58da 100644 --- a/distro/CHANGELOG +++ b/distro/CHANGELOG @@ -1,3 +1,5 @@ +v 1.1.1 (06/10/2014) + * added storage status report filter v 1.1.0 (08/09/2014) * added storage status report v. 1.0.2 (30-06-2014) diff --git a/distro/changelog.xml b/distro/changelog.xml index 0bca5b5..52f619a 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -1,6 +1,9 @@ - + + added storage status report filter + + added storage status report diff --git a/pom.xml b/pom.xml index 8158bf4..6bc8f89 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.gcube.contentmanagement storage-manager-trigger - 1.1.0-SNAPSHOT + 1.1.1-SNAPSHOT scm:svn:http://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/${project.artifactId} scm:svn:https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/${project.artifactId} diff --git a/src/main/java/org/gcube/contentmanager/storageserver/accounting/Report.java b/src/main/java/org/gcube/contentmanager/storageserver/accounting/Report.java index c7658f6..9f1ebba 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/accounting/Report.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/accounting/Report.java @@ -26,5 +26,10 @@ public interface Report { * @return */ public void send(RawUsageRecord sur); + + /** + * + */ + public abstract void printRecord(RawUsageRecord record); } diff --git a/src/main/java/org/gcube/contentmanager/storageserver/accounting/ReportAccountingImpl.java b/src/main/java/org/gcube/contentmanager/storageserver/accounting/ReportAccountingImpl.java index 07d4527..e1f580a 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/accounting/ReportAccountingImpl.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/accounting/ReportAccountingImpl.java @@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory; @Override public RawUsageRecord setGenericProperties(String resourceType, String consumerId, String resourceScope, String creationTime, String lastAccess, String owner) { - logger.info("set accounting generic properties: resourceType: "+resourceType+" consumerId "+consumerId+" scope: "+resourceScope+ " creationTime "+creationTime+" lastAccess "+lastAccess+" owner "+ owner); + logger.trace("set accounting generic properties: resourceType: "+resourceType+" consumerId "+consumerId+" scope: "+resourceScope+ " creationTime "+creationTime+" lastAccess "+lastAccess+" owner "+ owner); if(raFactory==null) init(); RawUsageRecord sr = new RawUsageRecord(); sr.setResourceType(resourceType); @@ -67,7 +67,7 @@ import org.slf4j.LoggerFactory; @Override public RawUsageRecord setSpecificProperties( RawUsageRecord sur, String operation, String size, String filePath, String callerIP, String dataType, String dataCount) { - logger.info("set accounting properties: operation: "+operation+" size: "+size+ " remotePath: "+filePath+" callerIP "+callerIP+" dataType "+dataType+" dataCount "+dataCount); + logger.trace("set accounting properties: operation: "+operation+" size: "+size+ " remotePath: "+filePath+" callerIP "+callerIP+" dataType "+dataType+" dataCount "+dataCount); if(sur==null) sur = new RawUsageRecord(); if (operation!=null) sur.setResourceSpecificProperty("operationType",operation); if(size!= null) sur.setResourceSpecificProperty("dataVolume", size); @@ -77,6 +77,22 @@ import org.slf4j.LoggerFactory; sur.setResourceSpecificProperty("dataCount", dataCount); return sur; } + + public void printRecord(RawUsageRecord record){ + logger.info(" accounting properties: " + + "\n\t owner: "+record.getResourceOwner()+ + "\n\t scope "+record.getResourceScope()+ + "\n\t type "+record.getResourceType()+ + "\n\t consumer "+record.getConsumerId()+ + "\n\t startTime "+record.getStartTime()+ + "\n\t endTime "+record.getEndTime()+ + "\n\t file "+record.getResourceSpecificProperty("remotePath")+ + "\n\t size "+record.getResourceSpecificProperty("dataVolume")+ + "\n\t caller "+record.getResourceSpecificProperty("callerIP")+ + "\n\t dataType "+record.getResourceSpecificProperty("dataType")+ + "\n\t dataCount "+record.getResourceSpecificProperty("dataCount")+ + "\n\t operation "+record.getResourceSpecificProperty("operationType")); + } @Override public void send(RawUsageRecord sur) { @@ -86,9 +102,9 @@ import org.slf4j.LoggerFactory; e.printStackTrace(); } if(raFactory!=null){ - logger.info("report sending..."); + logger.debug("report sending..."); raFactory.sendAccountingMessage(sur); - logger.info(" report send "); + logger.info(" report send: \n\t"+sur); }else logger.error("Problem on building accounting record: Factory Object is null "); } diff --git a/src/main/java/org/gcube/contentmanager/storageserver/parse/JsonParser.java b/src/main/java/org/gcube/contentmanager/storageserver/parse/JsonParser.java index ee089bb..7e4a620 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/parse/JsonParser.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/parse/JsonParser.java @@ -109,21 +109,28 @@ public class JsonParser extends Thread{ logger.warn("lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next "); } StorageStatusRecord ssr=null; - try{ - mongo=new MongoDB(server, user, password); - ssr=mongo.update(lastUser, length, 1, lastOperation); - mongo.close(); - }catch(Exception e){ - logger.error("Problem in updating storage status record: "+e.getMessage()); + if(isNeedSSReport(lastOperation)){ + try{ + mongo=new MongoDB(server, user, password); + if(lastOperation.equalsIgnoreCase("COPY")) + owner=lastUser; + ssr=mongo.update(owner, length, 1, lastOperation); + mongo.close(); + }catch(Exception e){ + logger.error("Problem in updating storage status record: "+e.getMessage()); + } } try{ logger.info(" operation accounted "+lastOperation); //call to the accounting library - report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, ssr.getVolume()+"", ssr.getCount()+""); + if (ssr !=null) + report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, ssr.getVolume()+"", ssr.getCount()+""); + else + report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, null, null); }catch(Exception e){ logger.error("Problem sending accounting report. Exception message: "+e.getMessage()); } - logger.info("\n[accountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp); +// logger.info("\n[accountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp); }else{ logger.info("operation is not accounted: invalid scope: "+scope); } @@ -142,6 +149,12 @@ public class JsonParser extends Thread{ } + private boolean isNeedSSReport(String lastOperation) { + if(lastOperation.equalsIgnoreCase("UPLOAD") || lastOperation.equalsIgnoreCase("COPY") || lastOperation.equalsIgnoreCase("DELETE")) + return true; + return false; + } + public void runWithoutThread(DBObject x){ try { report=new ReportFactory().getReport(ReportConfig.ACCOUNTING_TYPE); @@ -184,14 +197,22 @@ public class JsonParser extends Thread{ }else if((lastOperation==null) || (lastUser==null)){ logger.warn("lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next "); } -// operation=mappingOperationField(op, id, delete, lastAccess); + logger.debug(" current operation "+lastOperation); StorageStatusRecord ssr=null; - try{ - mongo=new MongoDB(server, user, password); - ssr=mongo.update(lastUser, length, 1, lastOperation); - mongo.close(); - logger.info(" operation accounted "+lastOperation); - logger.info("\n[accountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp); + if(isNeedSSReport(lastOperation)){ + try{ + mongo=new MongoDB(server, user, password); + if(lastOperation.equalsIgnoreCase("COPY")){ + ssr=mongo.update(lastUser, length, 1, lastOperation); + }else{ + ssr=mongo.update(owner, length, 1, lastOperation); + } + mongo.close(); + }catch(Exception e){ + logger.error("Problem in updating storage status record: "+e.getMessage()); + } + } + try{ //call to the accounting library report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, ssr.getVolume()+"", ssr.getCount()+""); }catch(Exception e){ @@ -242,15 +263,22 @@ public class JsonParser extends Thread{ RawUsageRecord sur=report.setGenericProperties("storage-usage",lastUser, scope, creationTime, lastAccess, owner); sur=report.setSpecificProperties(sur, operation, length+"", filename, callerIP, "STORAGE", "1"); logger.info("[accounting call] type: storage usage "); + report.printRecord(sur); report.send(sur); // ACCOUNTING CALL TYPE: STORAGE STATUS logger.debug("set properties: totVolume: "+totVolume+" totCount "+totCount); - RawUsageRecord ssr=report.setGenericProperties("storage-status",lastUser, scope, creationTime, lastAccess, owner); - ssr=report.setSpecificProperties(ssr, operation, totVolume, filename, callerIP, "STORAGE", totCount); - logger.info("[accounting call] type: storage status "); - report.send(ssr); - + if((totVolume!=null) && (totCount!=null)){ + RawUsageRecord ssr=report.setGenericProperties("storage-status",lastUser, scope, creationTime, lastAccess, owner); + ssr=report.setSpecificProperties(ssr, operation, totVolume, filename, callerIP, "STORAGE", totCount); + logger.info("[accounting call] type: storage status "); + report.printRecord(ssr); + report.send(ssr); + }else{ + logger.info("StorageStatus record not send for operation "+lastOperation); + } } + + private String retrieveScopeFromRemoteFilePath(String filename) { @@ -273,6 +301,17 @@ public class JsonParser extends Thread{ return null; } +// private StorageStatusRecord getStorageStatusRecord(String[] server, String user, String password, String owner, String lastUser, long length, String lastOperation, MongoDB mongo){ +// StorageStatusRecord ssr=null; +// try{ +// mongo=new MongoDB(server, user, password); +// ssr=mongo.update(lastUser, length, 1, lastOperation); +// mongo.close(); +// }catch(Exception e){ +// logger.error("Problem in updating storage status record: "+e.getMessage()); +// } +// return ssr; +// } } diff --git a/src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java b/src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java index 7f8b84f..f3d5458 100644 --- a/src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java +++ b/src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java @@ -19,6 +19,10 @@ public class Startup { public static void main(String[] args) { + + for (int i=0; i