gCubeSystem
/
storage-manager-trigger
16
0
Fork 0
Code Issues Releases Wiki Activity

added storage status report filter 

update to version 1.1.1-SNAPSHOT

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/storage-manager-trigger@100491 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
roberto.cirillo 2014-10-07 15:06:09 +00:00
parent 1b7c905ecb
commit 23561c3a80
8 changed files with 97 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
distro/CHANGELOG
View File

@ -1,3 +1,5 @@
v 1.1.1 (06/10/2014)
* added storage status report filter
v 1.1.0 (08/09/2014)
* added storage status report
v. 1.0.2 (30-06-2014)

5
distro/changelog.xml
View File

@ -1,6 +1,9 @@
<ReleaseNotes>
<ReleaseNotes>
<Changeset component="org.gcube.content-management.storage-manager-trigger.1-1-0" date="2014-06-30">
<Changeset component="org.gcube.content-management.storage-manager-trigger.1-1-1" date="2014-10-06">
<Change>added storage status report filter</Change>
</Changeset>
<Changeset component="org.gcube.content-management.storage-manager-trigger.1-1-0" date="2014-09-08">
<Change>added storage status report</Change>
</Changeset>
<Changeset component="org.gcube.content-management.storage-manager-trigger.1-0-2" date="2014-06-30">

2
pom.xml
View File

@ -8,7 +8,7 @@
</parent>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-trigger</artifactId>
<version>1.1.0-SNAPSHOT</version>
<version>1.1.1-SNAPSHOT</version>
<scm>
<connection>scm:svn:http://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/${project.artifactId}</connection>
<developerConnection>scm:svn:https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/${project.artifactId}</developerConnection>

5
src/main/java/org/gcube/contentmanager/storageserver/accounting/Report.java
View File

@ -26,5 +26,10 @@ public interface Report {
* @return
*/
public void send(RawUsageRecord sur);
/**
*
*/
public abstract void printRecord(RawUsageRecord record);
}

24
src/main/java/org/gcube/contentmanager/storageserver/accounting/ReportAccountingImpl.java
View File

@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
@Override
public RawUsageRecord setGenericProperties(String resourceType, String consumerId, String resourceScope, String creationTime, String lastAccess, String owner) {
logger.info("set accounting generic properties: resourceType: "+resourceType+" consumerId "+consumerId+" scope: "+resourceScope+ " creationTime "+creationTime+" lastAccess "+lastAccess+" owner "+ owner);
logger.trace("set accounting generic properties: resourceType: "+resourceType+" consumerId "+consumerId+" scope: "+resourceScope+ " creationTime "+creationTime+" lastAccess "+lastAccess+" owner "+ owner);
if(raFactory==null) init();
RawUsageRecord sr = new RawUsageRecord();
sr.setResourceType(resourceType);
@ -67,7 +67,7 @@ import org.slf4j.LoggerFactory;
@Override
public RawUsageRecord setSpecificProperties( RawUsageRecord sur, String operation, String size, String filePath, String callerIP, String dataType, String dataCount) {
logger.info("set accounting properties: operation: "+operation+" size: "+size+ " remotePath: "+filePath+" callerIP "+callerIP+" dataType "+dataType+" dataCount "+dataCount);
logger.trace("set accounting properties: operation: "+operation+" size: "+size+ " remotePath: "+filePath+" callerIP "+callerIP+" dataType "+dataType+" dataCount "+dataCount);
if(sur==null) sur = new RawUsageRecord();
if (operation!=null) sur.setResourceSpecificProperty("operationType",operation);
if(size!= null) sur.setResourceSpecificProperty("dataVolume", size);
@ -77,6 +77,22 @@ import org.slf4j.LoggerFactory;
sur.setResourceSpecificProperty("dataCount", dataCount);
return sur;
}
public void printRecord(RawUsageRecord record){
logger.info(" accounting properties: " +
"\n\t owner: "+record.getResourceOwner()+
"\n\t scope "+record.getResourceScope()+
"\n\t type "+record.getResourceType()+
"\n\t consumer "+record.getConsumerId()+
"\n\t startTime "+record.getStartTime()+
"\n\t endTime "+record.getEndTime()+
"\n\t file "+record.getResourceSpecificProperty("remotePath")+
"\n\t size "+record.getResourceSpecificProperty("dataVolume")+
"\n\t caller "+record.getResourceSpecificProperty("callerIP")+
"\n\t dataType "+record.getResourceSpecificProperty("dataType")+
"\n\t dataCount "+record.getResourceSpecificProperty("dataCount")+
"\n\t operation "+record.getResourceSpecificProperty("operationType"));
}
@Override
public void send(RawUsageRecord sur) {
@ -86,9 +102,9 @@ import org.slf4j.LoggerFactory;
e.printStackTrace();
}
if(raFactory!=null){
logger.info("report sending...");
logger.debug("report sending...");
raFactory.sendAccountingMessage(sur);
logger.info(" report send ");
logger.info(" report send: \n\t"+sur);
}else
logger.error("Problem on building accounting record: Factory Object is null ");
}

79
src/main/java/org/gcube/contentmanager/storageserver/parse/JsonParser.java
View File

@ -109,21 +109,28 @@ public class JsonParser extends Thread{
logger.warn("lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next ");
}
StorageStatusRecord ssr=null;
try{
mongo=new MongoDB(server, user, password);
ssr=mongo.update(lastUser, length, 1, lastOperation);
mongo.close();
}catch(Exception e){
logger.error("Problem in updating storage status record: "+e.getMessage());
if(isNeedSSReport(lastOperation)){
try{
mongo=new MongoDB(server, user, password);
if(lastOperation.equalsIgnoreCase("COPY"))
owner=lastUser;
ssr=mongo.update(owner, length, 1, lastOperation);
mongo.close();
}catch(Exception e){
logger.error("Problem in updating storage status record: "+e.getMessage());
}
}
try{
logger.info(" operation accounted "+lastOperation);
//call to the accounting library
report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, ssr.getVolume()+"", ssr.getCount()+"");
if (ssr !=null)
report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, ssr.getVolume()+"", ssr.getCount()+"");
else
report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, null, null);
}catch(Exception e){
logger.error("Problem sending accounting report. Exception message: "+e.getMessage());
}
logger.info("\n[accountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp);
// logger.info("\n[accountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp);
}else{
logger.info("operation is not accounted: invalid scope: "+scope);
}
@ -142,6 +149,12 @@ public class JsonParser extends Thread{
}
private boolean isNeedSSReport(String lastOperation) {
if(lastOperation.equalsIgnoreCase("UPLOAD") || lastOperation.equalsIgnoreCase("COPY") || lastOperation.equalsIgnoreCase("DELETE"))
return true;
return false;
}
public void runWithoutThread(DBObject x){
try {
report=new ReportFactory().getReport(ReportConfig.ACCOUNTING_TYPE);
@ -184,14 +197,22 @@ public class JsonParser extends Thread{
}else if((lastOperation==null) || (lastUser==null)){
logger.warn("lastOperation: "+lastOperation+" lastUser: "+lastUser+". These values cannot be null. Skip next ");
}
// operation=mappingOperationField(op, id, delete, lastAccess);
logger.debug(" current operation "+lastOperation);
StorageStatusRecord ssr=null;
try{
mongo=new MongoDB(server, user, password);
ssr=mongo.update(lastUser, length, 1, lastOperation);
mongo.close();
logger.info(" operation accounted "+lastOperation);
logger.info("\n[accountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp);
if(isNeedSSReport(lastOperation)){
try{
mongo=new MongoDB(server, user, password);
if(lastOperation.equalsIgnoreCase("COPY")){
ssr=mongo.update(lastUser, length, 1, lastOperation);
}else{
ssr=mongo.update(owner, length, 1, lastOperation);
}
mongo.close();
}catch(Exception e){
logger.error("Problem in updating storage status record: "+e.getMessage());
}
}
try{
//call to the accounting library
report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, ssr.getVolume()+"", ssr.getCount()+"");
}catch(Exception e){
@ -242,15 +263,22 @@ public class JsonParser extends Thread{
RawUsageRecord sur=report.setGenericProperties("storage-usage",lastUser, scope, creationTime, lastAccess, owner);
sur=report.setSpecificProperties(sur, operation, length+"", filename, callerIP, "STORAGE", "1");
logger.info("[accounting call] type: storage usage ");
report.printRecord(sur);
report.send(sur);
// ACCOUNTING CALL TYPE: STORAGE STATUS
logger.debug("set properties: totVolume: "+totVolume+" totCount "+totCount);
RawUsageRecord ssr=report.setGenericProperties("storage-status",lastUser, scope, creationTime, lastAccess, owner);
ssr=report.setSpecificProperties(ssr, operation, totVolume, filename, callerIP, "STORAGE", totCount);
logger.info("[accounting call] type: storage status ");
report.send(ssr);
if((totVolume!=null) && (totCount!=null)){
RawUsageRecord ssr=report.setGenericProperties("storage-status",lastUser, scope, creationTime, lastAccess, owner);
ssr=report.setSpecificProperties(ssr, operation, totVolume, filename, callerIP, "STORAGE", totCount);
logger.info("[accounting call] type: storage status ");
report.printRecord(ssr);
report.send(ssr);
}else{
logger.info("StorageStatus record not send for operation "+lastOperation);
}
}
private String retrieveScopeFromRemoteFilePath(String filename) {
@ -273,6 +301,17 @@ public class JsonParser extends Thread{
return null;
}
// private StorageStatusRecord getStorageStatusRecord(String[] server, String user, String password, String owner, String lastUser, long length, String lastOperation, MongoDB mongo){
// StorageStatusRecord ssr=null;
// try{
// mongo=new MongoDB(server, user, password);
// ssr=mongo.update(lastUser, length, 1, lastOperation);
// mongo.close();
// }catch(Exception e){
// logger.error("Problem in updating storage status record: "+e.getMessage());
// }
// return ssr;
// }
}

5
src/main/java/org/gcube/contentmanager/storageserver/startup/Startup.java
View File

@ -19,6 +19,10 @@ public class Startup {
public static void main(String[] args) {
for (int i=0; i<args.length;i++)
System.out.println("param N." +i + ": " + args[i]);
if(args.length != 2 && args.length != 4){
System.out.println("Usage:");
System.out.println("\tjava Startup scope ip user password\n\n");
@ -32,6 +36,7 @@ public class Startup {
return;
}
scope=args[0];
String oplogServer= args[1];
if(args.length == 4){

2
src/main/java/org/gcube/contentmanager/storageserver/store/MongoDB.java
View File

@ -132,7 +132,7 @@ public class MongoDB {
private int setCount(int count, int currentCount, String operation) {
logger.info("accounting: operation "+operation+" total count "+currentCount+" current count"+count);
if(operation.equalsIgnoreCase("UPLOAD"))
if(operation.equalsIgnoreCase("UPLOAD")|| operation.equalsIgnoreCase("COPY"))
currentCount=currentCount+count;
else if(operation.equalsIgnoreCase("DELETE"))
currentCount=currentCount-count;