2014-10-22 12:38:13 +02:00
|
|
|
package org.gcube.contentmanager.storageserver.consumer;
|
2014-03-13 15:22:33 +01:00
|
|
|
|
2014-10-10 16:32:23 +02:00
|
|
|
import java.util.List;
|
|
|
|
|
2015-07-29 15:02:19 +02:00
|
|
|
import org.bson.types.ObjectId;
|
|
|
|
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
|
|
|
|
import org.gcube.common.scope.api.ScopeProvider;
|
2014-03-13 15:22:33 +01:00
|
|
|
import org.gcube.contentmanager.storageserver.accounting.Report;
|
|
|
|
import org.gcube.contentmanager.storageserver.accounting.ReportConfig;
|
|
|
|
import org.gcube.contentmanager.storageserver.accounting.ReportException;
|
|
|
|
import org.gcube.contentmanager.storageserver.accounting.ReportFactory;
|
|
|
|
import org.gcube.contentmanager.storageserver.data.CubbyHole;
|
2015-07-29 15:02:19 +02:00
|
|
|
import org.gcube.contentmanager.storageserver.data.OpLogRemoteObject;
|
2014-04-16 10:54:47 +02:00
|
|
|
import org.gcube.contentmanager.storageserver.parse.utils.ValidationUtils;
|
2014-06-05 19:27:16 +02:00
|
|
|
import org.gcube.contentmanager.storageserver.store.MongoDB;
|
2014-10-22 17:47:24 +02:00
|
|
|
import org.gcube.contentmanager.storageserver.store.StorageStatusObject;
|
2014-03-13 15:22:33 +01:00
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
|
|
|
import com.mongodb.DBObject;
|
|
|
|
|
2014-10-22 12:38:13 +02:00
|
|
|
public class UserAccountingConsumer extends Thread{
|
2014-03-13 15:22:33 +01:00
|
|
|
|
2014-10-22 12:38:13 +02:00
|
|
|
final static Logger logger=LoggerFactory.getLogger(UserAccountingConsumer.class);
|
2014-03-21 16:50:43 +01:00
|
|
|
final static int MINUTE_DECREMENT=-2;
|
2014-03-13 15:22:33 +01:00
|
|
|
private CubbyHole c;
|
|
|
|
private int number;
|
|
|
|
private Report report;
|
2014-04-01 11:20:28 +02:00
|
|
|
// object fields
|
|
|
|
private String op;
|
2014-06-05 19:27:16 +02:00
|
|
|
private String user;
|
|
|
|
private String password;
|
|
|
|
String[] server;
|
2014-10-10 16:32:23 +02:00
|
|
|
List<String> dtsHosts;
|
2015-07-29 15:02:19 +02:00
|
|
|
// private String id;
|
2014-06-05 19:27:16 +02:00
|
|
|
|
2014-10-22 12:38:13 +02:00
|
|
|
public UserAccountingConsumer(String[] srvs, CubbyHole c, int number,List<String> dtsHosts){
|
2014-03-13 15:22:33 +01:00
|
|
|
this.c=c;
|
|
|
|
this.number=number;
|
2014-06-05 19:27:16 +02:00
|
|
|
this.server=srvs;
|
2014-10-10 16:32:23 +02:00
|
|
|
this.dtsHosts=dtsHosts;
|
2014-06-05 19:27:16 +02:00
|
|
|
// init the accounting report
|
|
|
|
try {
|
|
|
|
init();
|
|
|
|
} catch (ReportException e) {
|
|
|
|
throw new RuntimeException("Accounting report Exception initialization");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-22 12:38:13 +02:00
|
|
|
public UserAccountingConsumer(String[] srvs, String user, String password, CubbyHole c, int number, List<String> dtsHosts){
|
2014-06-05 19:27:16 +02:00
|
|
|
this.c=c;
|
|
|
|
this.number=number;
|
|
|
|
this.server=srvs;
|
2014-10-10 16:32:23 +02:00
|
|
|
this.dtsHosts=dtsHosts;
|
2014-06-05 19:27:16 +02:00
|
|
|
this.user=user;
|
|
|
|
this.password=password;
|
2014-03-19 18:08:15 +01:00
|
|
|
// init the accounting report
|
2014-03-13 15:22:33 +01:00
|
|
|
try {
|
|
|
|
init();
|
|
|
|
} catch (ReportException e) {
|
|
|
|
throw new RuntimeException("Accounting report Exception initialization");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-10 10:23:30 +02:00
|
|
|
|
2014-03-13 15:22:33 +01:00
|
|
|
private void init() throws ReportException{
|
2014-09-10 10:23:30 +02:00
|
|
|
report=ReportFactory.getReport(ReportConfig.ACCOUNTING_TYPE);
|
2014-04-15 17:57:57 +02:00
|
|
|
report.init();
|
2014-03-13 15:22:33 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
public void run() {
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("Consuming SU record started ");
|
2014-03-13 15:22:33 +01:00
|
|
|
while(true){
|
2014-06-09 10:10:13 +02:00
|
|
|
DBObject x=null;
|
2014-06-30 14:42:50 +02:00
|
|
|
MongoDB mongo=null;
|
2014-06-09 10:10:13 +02:00
|
|
|
try{
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("SU waiting next record... ");
|
2014-06-09 10:10:13 +02:00
|
|
|
x=c.get();
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.info("Consumer #" + this.number + " got: " + x.get("_id"));
|
|
|
|
|
|
|
|
//get operation
|
2014-06-09 10:10:13 +02:00
|
|
|
op = (String) x.get("op");
|
|
|
|
// retrieve object fields
|
|
|
|
DBObject obj=(DBObject)x.get("o");
|
2015-07-29 15:02:19 +02:00
|
|
|
OpLogRemoteObject record=retrieveObjectFields(obj);
|
2014-09-10 10:23:30 +02:00
|
|
|
// set object dimension
|
2015-07-29 15:02:19 +02:00
|
|
|
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("[recordCheck] operation: "+op+"\n\t name: "+record.getName()+"\n\t type: "+record.getType()+"\n\t path: "+record.getFilename()+"\n\t dir Path: "+record.getDir()+"\n\t length: "+record.getLength()+"\n\t owner: "+record.getOwner()+ "\n\t lastOperation "+record.getLastOperation()+"\n\t lastUser: "+record.getLastUser()+"\n\t lastAccess: "+record.getLastAccess());
|
2015-10-09 11:32:18 +02:00
|
|
|
if(((record.getLength() >0) && (((record.getFilename() !=null) && (record.getFilename().length()>0) && (record.getDir().length()>0)&& (record.getDir().contains("/"))) || (record.getLinkCount() > 0)))){
|
2014-06-09 10:10:13 +02:00
|
|
|
//convert from byte to kb
|
2015-07-29 15:02:19 +02:00
|
|
|
record.setLength(record.getLength()/1024);
|
2014-09-10 10:23:30 +02:00
|
|
|
// check scope
|
2014-06-09 10:10:13 +02:00
|
|
|
String scope=null;
|
2015-10-09 11:32:18 +02:00
|
|
|
String pathString=(String)obj.get("onScope");
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("[recordCheck] pathString value: "+pathString);
|
2015-10-09 11:32:18 +02:00
|
|
|
if((record.getDir()!=null)&& (record.getDir().contains("/"))){
|
|
|
|
scope=retrieveScopeFromRemoteFilePath(record.getDir());
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("[recordCheck] scope retrieved: "+scope);
|
2015-10-09 11:32:18 +02:00
|
|
|
}else{
|
2014-06-09 10:10:13 +02:00
|
|
|
// field added on storage manager library for retrieve scope. Used only if it is a link delete
|
|
|
|
scope=retrieveScopeFromRemoteFilePath(pathString);
|
2014-04-16 10:54:47 +02:00
|
|
|
}
|
2014-06-09 10:10:13 +02:00
|
|
|
boolean validScope=ValidationUtils.validationScope(scope);
|
|
|
|
if(validScope){
|
2015-07-29 15:02:19 +02:00
|
|
|
if(record.getDelete() != null){
|
|
|
|
record.setLastOperation("DELETE");
|
|
|
|
}else if ((record.getLastOperation() != null) && (op != null) && (record.getLastOperation().equalsIgnoreCase("LINK")) && (op.equalsIgnoreCase("u"))){
|
2014-06-09 10:10:13 +02:00
|
|
|
// it is an update on a link object this operation doesn't be accounted
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.info("[recordCheck] update on link object is not accounted. Skip next ");
|
2014-06-09 10:10:13 +02:00
|
|
|
continue;
|
2015-07-29 15:02:19 +02:00
|
|
|
}else if((record.getLastOperation()==null) || (record.getLastUser()==null)){
|
|
|
|
logger.warn("lastOperation: "+record.getLastOperation()+" lastUser: "+record.getLastUser()+". These values cannot be null. Skip next ");
|
2014-10-22 12:38:13 +02:00
|
|
|
continue;
|
|
|
|
}else{
|
|
|
|
// if the lastoperation is download and the caller contains a dts host then it isn't a real user but it is a workspace accounting record
|
2015-07-29 15:02:19 +02:00
|
|
|
if((dtsHosts != null) && (record.getLastOperation() != null) && (record.getLastOperation().equalsIgnoreCase("DOWNLOAD"))){
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("[recordCheck] check if the caller is from dts. CallerIP: "+record.getCallerIp()+" dts value: "+dtsHosts);
|
2014-10-22 12:38:13 +02:00
|
|
|
for(String host: dtsHosts){
|
|
|
|
logger.debug("scan "+host);
|
2015-07-29 15:02:19 +02:00
|
|
|
if(record.getCallerIp().contains(host)){
|
|
|
|
record.setLastOperation("workspace.accounting");
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("[recordCheck] the caller is dts service: caller "+record.getCallerIp()+ " dts host: "+host+" the new user is: "+record.getLastUser());
|
2014-10-22 12:38:13 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-10-10 16:32:23 +02:00
|
|
|
}
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("[recordCheck] operation accounted "+record.getLastOperation());
|
2014-10-22 17:47:24 +02:00
|
|
|
StorageStatusObject ssr=null;
|
2015-07-29 15:02:19 +02:00
|
|
|
if(isNeedSSReport(record.getLastOperation())){
|
2014-10-07 17:06:09 +02:00
|
|
|
try{
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("[recordCheck] update SS record");
|
2014-10-07 17:06:09 +02:00
|
|
|
mongo=new MongoDB(server, user, password);
|
2015-07-29 15:02:19 +02:00
|
|
|
if(record.getLastOperation().equalsIgnoreCase("COPY"))
|
|
|
|
record.setOwner(record.getLastUser());
|
|
|
|
ssr=new StorageStatusObject(record.getOwner(), record.getLength(), 1);
|
|
|
|
ssr=mongo.updateUserVolume(ssr, record.getLastOperation());
|
2014-10-07 17:06:09 +02:00
|
|
|
mongo.close();
|
|
|
|
}catch(Exception e){
|
2015-10-14 11:48:30 +02:00
|
|
|
mongo.close();
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.error("[recordCheck] Problem in updating storage status record: "+e.getMessage());
|
2014-10-07 17:06:09 +02:00
|
|
|
}
|
2014-09-10 10:23:30 +02:00
|
|
|
}
|
|
|
|
try{
|
2015-07-29 15:02:19 +02:00
|
|
|
// scope set add in 1.3 for new accounting libs
|
|
|
|
ScopeProvider.instance.set(scope);
|
2014-09-10 10:23:30 +02:00
|
|
|
//call to the accounting library
|
2014-10-07 17:06:09 +02:00
|
|
|
if (ssr !=null)
|
2015-07-29 15:02:19 +02:00
|
|
|
report( record, scope, ssr.getVolume()+"", ssr.getCount()+"");
|
2014-10-07 17:06:09 +02:00
|
|
|
else
|
2015-07-29 15:02:19 +02:00
|
|
|
report( record, scope, null, null);
|
2014-09-10 10:23:30 +02:00
|
|
|
}catch(Exception e){
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.error("[recordCheck] Problem sending accounting report. Exception message: "+e.getMessage());
|
2014-06-09 10:10:13 +02:00
|
|
|
}
|
|
|
|
}else{
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.info("[recordCheck] operation "+record.getLastOperation()+" is not accounted: invalid scope: "+scope);
|
2014-06-09 10:10:13 +02:00
|
|
|
}
|
|
|
|
}else{
|
2015-07-29 15:02:19 +02:00
|
|
|
logger.info("operation "+record.getLastOperation()+" is not accounted");
|
2014-06-09 10:10:13 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
}catch(Exception e){
|
2016-11-11 12:03:52 +01:00
|
|
|
logger.error(" "+e.getCause().getLocalizedMessage());
|
|
|
|
logger.error("ERROR Processing record: "+x+" Exception throws: "+e.getMessage());
|
2014-06-09 10:10:13 +02:00
|
|
|
logger.info("skip to next record ");
|
2014-06-30 14:42:50 +02:00
|
|
|
if(mongo!=null)
|
|
|
|
mongo.close();
|
2014-06-09 10:10:13 +02:00
|
|
|
}
|
2014-03-13 15:22:33 +01:00
|
|
|
}
|
|
|
|
}
|
2014-09-10 10:23:30 +02:00
|
|
|
|
2014-10-22 12:38:13 +02:00
|
|
|
/**
|
|
|
|
* check if the lastOperation is a valid operation for storage status information record
|
|
|
|
* @param lastOperation
|
|
|
|
* @return
|
|
|
|
*/
|
2014-10-07 17:06:09 +02:00
|
|
|
private boolean isNeedSSReport(String lastOperation) {
|
2016-11-17 11:12:36 +01:00
|
|
|
logger.debug("Last operation is "+lastOperation);
|
2014-10-07 17:06:09 +02:00
|
|
|
if(lastOperation.equalsIgnoreCase("UPLOAD") || lastOperation.equalsIgnoreCase("COPY") || lastOperation.equalsIgnoreCase("DELETE"))
|
|
|
|
return true;
|
|
|
|
return false;
|
|
|
|
}
|
2014-10-22 12:38:13 +02:00
|
|
|
|
2014-10-07 17:06:09 +02:00
|
|
|
|
2014-09-10 10:23:30 +02:00
|
|
|
public void runWithoutThread(DBObject x){
|
|
|
|
try {
|
2014-10-10 16:32:23 +02:00
|
|
|
report=ReportFactory.getReport(ReportConfig.ACCOUNTING_TYPE);
|
2014-09-10 10:23:30 +02:00
|
|
|
} catch (ReportException e1) {
|
|
|
|
// TODO Auto-generated catch block
|
|
|
|
e1.printStackTrace();
|
|
|
|
}
|
|
|
|
report.init();
|
2014-10-13 12:24:19 +02:00
|
|
|
run();
|
2014-09-10 10:23:30 +02:00
|
|
|
}
|
2014-03-13 15:22:33 +01:00
|
|
|
|
2015-07-29 15:02:19 +02:00
|
|
|
private OpLogRemoteObject retrieveObjectFields(DBObject obj) {
|
|
|
|
OpLogRemoteObject record = new OpLogRemoteObject();
|
|
|
|
record.setFilename((String) obj.get("filename"));
|
|
|
|
record.setType((String) obj.get("type"));
|
|
|
|
record.setName( (String) obj.get("name"));
|
|
|
|
record.setOwner((String) obj.get("owner"));
|
|
|
|
record.setCreationTime((String) obj.get("creationTime"));
|
2015-10-09 11:32:18 +02:00
|
|
|
record.setDir((String) obj.get("dir"));
|
2015-07-29 15:02:19 +02:00
|
|
|
if(obj.get("lastAccess") != null) record.setLastAccess((String)obj.get("lastAccess"));
|
|
|
|
if(obj.get("callerIP") != null) record.setCallerIp((String)obj.get("callerIP"));
|
|
|
|
if(obj.get("lastOperation") != null) record.setLastOperation((String)obj.get("lastOperation"));
|
|
|
|
if(obj.get("lastUser") != null) record.setLastUser((String)obj.get("lastUser"));
|
|
|
|
if(obj.get("linkCount") != null) record.setLinkCount((int)obj.get("linkCount"));
|
|
|
|
if(obj.get("onDeleting") != null) record.setDelete((String)obj.get("onDeleting"));
|
|
|
|
if(obj.get("_id") != null){
|
|
|
|
ObjectId id= (ObjectId)obj.get("_id");
|
|
|
|
if(ObjectId.isValid(id.toString()))
|
|
|
|
record.setId(id.toString());
|
|
|
|
}
|
|
|
|
long length=-1;
|
|
|
|
if(obj.get("length")!=null) record.setLength((long)obj.get("length"));
|
|
|
|
return record;
|
|
|
|
|
2014-03-19 18:08:15 +01:00
|
|
|
}
|
|
|
|
|
2015-07-29 15:02:19 +02:00
|
|
|
private void report(OpLogRemoteObject record, String scope, String totVolume, String totCount) {
|
2014-06-05 19:27:16 +02:00
|
|
|
// ACCOUNTING CALL TYPE: STORAGE USAGE
|
2015-10-14 11:48:30 +02:00
|
|
|
StorageUsageRecord sur=report.setGenericProperties(null, "storage-usage", record.getLastUser(), scope, record.getCreationTime(), record.getLastAccess(), record.getOwner(), record.getLastOperation(), record.getLength()+"");
|
2015-07-29 15:02:19 +02:00
|
|
|
sur=report.setSpecificProperties(sur, record.getFilename(), "STORAGE", record.getCallerIp(), record.getId());
|
2014-09-10 10:23:30 +02:00
|
|
|
logger.info("[accounting call] type: storage usage ");
|
2014-10-07 17:06:09 +02:00
|
|
|
report.printRecord(sur);
|
2014-06-05 19:27:16 +02:00
|
|
|
report.send(sur);
|
2014-03-13 15:22:33 +01:00
|
|
|
}
|
2014-10-07 17:06:09 +02:00
|
|
|
|
|
|
|
|
2014-03-13 15:22:33 +01:00
|
|
|
|
2014-04-01 11:20:28 +02:00
|
|
|
|
2014-03-19 18:08:15 +01:00
|
|
|
private String retrieveScopeFromRemoteFilePath(String filename) {
|
2014-03-13 15:22:33 +01:00
|
|
|
String[] split=filename.split("/");
|
|
|
|
if(split.length>0){
|
|
|
|
String scope=null;
|
|
|
|
int i=1;
|
|
|
|
if(split[1].equals("VOLATILE")){
|
|
|
|
i=2;
|
|
|
|
}
|
|
|
|
scope="/"+split[i];
|
|
|
|
i++;
|
|
|
|
while((!split[i].equals("home")) && (!split[i].equals("public"))){
|
|
|
|
scope=scope+"/"+split[i];
|
|
|
|
i++;
|
|
|
|
}
|
2014-10-23 18:08:31 +02:00
|
|
|
logger.debug("retieved scope: "+scope);
|
2014-03-13 15:22:33 +01:00
|
|
|
return scope;
|
|
|
|
}else logger.error("Scope bad format: scope not retrieved from string: "+filename);
|
|
|
|
return null;
|
|
|
|
}
|
2014-10-22 12:38:13 +02:00
|
|
|
|
2014-03-13 15:22:33 +01:00
|
|
|
}
|