gCubeSystem
/
storage-manager-trigger
16
0
Fork 0
Code Issues Releases Wiki Activity

added patch on deleting operation. It works only with storage-manager 2.1.0 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/storage-manager-trigger@93135 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
roberto.cirillo 2014-03-14 16:09:55 +00:00
parent 6001d5a0b4
commit 07395c202f
5 changed files with 2434 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2369
log.txt
View File

File diff suppressed because it is too large Load Diff

2
src/main/java/org/gcube/contentmanager/storageserver/accounting/Report.java
View File

@ -9,7 +9,7 @@ public interface Report {
* @param resourceScope
* @return
*/
public void init(String consumerId, String resourceScope);
public void init(String consumerId, String resourceScope, String creationTime);
/**
* set start time of the operation
* @return

57
src/main/java/org/gcube/contentmanager/storageserver/accounting/ReportAccountingImpl.java
View File

@ -4,6 +4,8 @@ package org.gcube.contentmanager.storageserver.accounting;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
@ -21,8 +23,8 @@ import org.slf4j.LoggerFactory;
public ResourceAccounting raFactory;
@Override
public void init(String consumerId, String resourceScope) {
logger.info("set accounting properties: consumerId "+consumerId+" scope: "+resourceScope);
public void init(String consumerId, String resourceScope, String creationTime) {
logger.info("set accounting properties: consumerId "+consumerId+" scope: "+resourceScope+ " creationTime "+creationTime);
raFactory = null;
try {
raFactory = ResourceAccountingFactory.getResourceAccountingInstance();
@ -39,9 +41,21 @@ import org.slf4j.LoggerFactory;
if(consumerId!=null) ur.setConsumerId(consumerId);
// ur.setResourceOwner("paolo.fabriani");
if(resourceScope !=null) ur.setResourceScope(resourceScope);
//set creation time
Calendar createTime = new GregorianCalendar();
ur.setCreateTime(createTime.getTime());
//set creation time
if(creationTime == null){
Calendar createTime = new GregorianCalendar();
ur.setCreateTime(createTime.getTime());
}else{
SimpleDateFormat formatter = new SimpleDateFormat("dd MM yyyy 'at' hh:mm:ss z");
Date date=null;
try {
date = formatter.parse(creationTime);
} catch (ParseException e) {
logger.error("Error in parsing date: "+e.getMessage());
e.printStackTrace();
}
ur.setCreateTime(date);
}
}
@ -53,8 +67,7 @@ import org.slf4j.LoggerFactory;
@Override
public void ultimate(String owner, String uri, String operation, String size, String filePath, String id) {
setEndTime();
logger.info("set accounting properties: owner "+owner+" uri: "+uri+" operation: "+operation+" size: "+size);
logger.info("set accounting properties: owner "+owner+" uri: "+uri+" operation: "+operation+" size: "+size+ " remotePath: "+filePath+" id: "+id);
//specific properties TODO
if(owner != null) ur.setResourceOwner(owner);
if(uri != null) ur.setResourceSpecificProperty("objectURI", uri);
@ -63,24 +76,25 @@ import org.slf4j.LoggerFactory;
if(size!= null) ur.setResourceSpecificProperty("dataVolume", size);
if(filePath != null) ur.setResourceSpecificProperty("remotePath", filePath);
if(id!= null) ur.setResourceSpecificProperty("id", id);
setEndTime();
}
@Override
public void send() {
logger.info("report sending ");
logger.info("report sending...");
raFactory.sendAccountingMessage(ur);
}
// private void setIpAddress() {
// String address=null;
// try {
// address=InetAddress.getLocalHost().toString();
// } catch (UnknownHostException e) {
//
// }
// logger.info("caller ip: "+address);
// if (address!=null) ur.setResourceSpecificProperty("callerIP", address);;
// }
@Deprecated
private void setIpAddress() {
String address=null;
try {
address=InetAddress.getLocalHost().toString();
} catch (UnknownHostException e) {
}
logger.info("caller ip: "+address);
if (address!=null) ur.setResourceSpecificProperty("callerIP", address);;
}
private void setStartTime() {
//set start time
@ -98,6 +112,9 @@ import org.slf4j.LoggerFactory;
// set end time
Calendar endTime = new GregorianCalendar();
Date time=endTime.getTime();
SimpleDateFormat sdf=new SimpleDateFormat();
sdf.applyPattern("dd MM yyyy 'at' hh:mm:ss z");//format(time);
sdf.format(time);
logger.info("set end time: "+time);
try {
ur.setEndTime(time);

6
src/main/java/org/gcube/contentmanager/storageserver/data/ReadingMongoOplog.java
View File

@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
public class ReadingMongoOplog extends Thread{
final static Logger logger=LoggerFactory.getLogger(ReadingMongoOplog.class);
final static String DBNAME="remotefs";
private ServerAddress[] server;
private MongoClient mongoClient;
private DB local;
@ -73,8 +74,11 @@ public class ReadingMongoOplog extends Thread{
cursor.addOption(Bytes.QUERYOPTION_AWAITDATA);
while (cursor.hasNext()) {
DBObject x = cursor.next();
logger.info("oplog current object: "+x);
ts = (BSONTimestamp) x.get("ts");
if(x.get("o2")!=null){
String ns=(String)x.get("ns");
// check if discard or process the current DB record
if((x.get("o2")!=null) || (ns.equalsIgnoreCase(DBNAME+".fs.files"))){
if(x.containsField("o")){
// parser.jsonRecordParser(x);
c.put(x);

34
src/main/java/org/gcube/contentmanager/storageserver/parse/JsonParser.java
View File

@ -17,6 +17,7 @@ public class JsonParser extends Thread{
private CubbyHole c;
private int number;
private Report report;
private String previousInsert;
public JsonParser(CubbyHole c, int number){
this.c=c;
this.number=number;
@ -43,29 +44,42 @@ public class JsonParser extends Thread{
String type=(String) obj.get("type");
String name=(String) obj.get("name");
String owner=(String) obj.get("owner");
String creationTime=(String) obj.get("creationTime");
String delete=null;
logger.info("obj.get(OnDeleting) "+obj.get("onDeleting"));
if(obj.get("onDeleting") != null) delete=(String)obj.get("onDeleting");
logger.info("delete field: "+delete);
ObjectId objectId=(ObjectId)obj.get("_id");
String id = objectId.toString();
long length=(long) obj.get("length");
long length=-1;
if(obj.get("length")!=null) length=(long)obj.get("length");
logger.debug("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+ " id: "+id);
if((length >0) && (name!=null)){
if((length >0) && (filename!=null)){
//call to the accounting library
String scope=retrieveScopeFromFilename(filename);
report.init(owner, scope);
report.timeUpdate();
String operation=mappingOperationField(op);
report.init(owner, scope, creationTime);
// report.timeUpdate();
String operation=mappingOperationField(op, id, delete);
report.ultimate(owner, null, operation, length+"", filename, id);
report.send();
logger.debug("[accountingCall] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner);
// }else if(op.equals("i")){
// previousInsert=id;
}else{
logger.debug("operation is not accounted");
logger.info("operation is not accounted");
}
}
}
private String mappingOperationField(String op) {
private String mappingOperationField(String op, String id, String onDeleting) {
logger.info("delete field: "+onDeleting);
if(op.equals("u")){
return "UPDATE";
if((onDeleting != null) && (onDeleting.equals("true"))){
logger.info("found delete field");
return "DELETE";
}else
return "UPDATE";
}else if(op.equals("i")){
return "INSERT";
}else if(op.equals("d")){
@ -81,10 +95,6 @@ public class JsonParser extends Thread{
int i=1;
if(split[1].equals("VOLATILE")){
i=2;
// scope="/"+split[1];
// if(split[2] != null && (!split[2].equals("home")) && (!split[2].equals("public"))){
// scope=scope+"/"+split[2];
// }
}
scope="/"+split[i];
i++;