gCubeSystem
/
storage-manager-trigger
16
0
Fork 0
Code Issues Releases Wiki Activity

added patch on deleting operation. It works only with storage-manager 2.1.0 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/storage-manager-trigger@93135 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
roberto.cirillo 2014-03-14 16:09:55 +00:00
parent 6001d5a0b4
commit 07395c202f
5 changed files with 2434 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2369
log.txt
View File

File diff suppressed because it is too large Load Diff

2
src/main/java/org/gcube/contentmanager/storageserver/accounting/Report.java
View File

@ -9,7 +9,7 @@ public interface Report {
* @param resourceScope * @param resourceScope
* @return * @return
*/ */
public void init(String consumerId, String resourceScope); public void init(String consumerId, String resourceScope, String creationTime);
/** /**
* set start time of the operation * set start time of the operation
* @return * @return

57
src/main/java/org/gcube/contentmanager/storageserver/accounting/ReportAccountingImpl.java
View File

@ -4,6 +4,8 @@ package org.gcube.contentmanager.storageserver.accounting;
import java.io.IOException; import java.io.IOException;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.GregorianCalendar; import java.util.GregorianCalendar;
@ -21,8 +23,8 @@ import org.slf4j.LoggerFactory;
public ResourceAccounting raFactory; public ResourceAccounting raFactory;
@Override @Override
public void init(String consumerId, String resourceScope) { public void init(String consumerId, String resourceScope, String creationTime) {
logger.info("set accounting properties: consumerId "+consumerId+" scope: "+resourceScope); logger.info("set accounting properties: consumerId "+consumerId+" scope: "+resourceScope+ " creationTime "+creationTime);
raFactory = null; raFactory = null;
try { try {
raFactory = ResourceAccountingFactory.getResourceAccountingInstance(); raFactory = ResourceAccountingFactory.getResourceAccountingInstance();
@ -39,9 +41,21 @@ import org.slf4j.LoggerFactory;
if(consumerId!=null) ur.setConsumerId(consumerId); if(consumerId!=null) ur.setConsumerId(consumerId);
// ur.setResourceOwner("paolo.fabriani"); // ur.setResourceOwner("paolo.fabriani");
if(resourceScope !=null) ur.setResourceScope(resourceScope); if(resourceScope !=null) ur.setResourceScope(resourceScope);
//set creation time //set creation time
Calendar createTime = new GregorianCalendar(); if(creationTime == null){
ur.setCreateTime(createTime.getTime()); Calendar createTime = new GregorianCalendar();
ur.setCreateTime(createTime.getTime());
}else{
SimpleDateFormat formatter = new SimpleDateFormat("dd MM yyyy 'at' hh:mm:ss z");
Date date=null;
try {
date = formatter.parse(creationTime);
} catch (ParseException e) {
logger.error("Error in parsing date: "+e.getMessage());
e.printStackTrace();
}
ur.setCreateTime(date);
}
} }
@ -53,8 +67,7 @@ import org.slf4j.LoggerFactory;
@Override @Override
public void ultimate(String owner, String uri, String operation, String size, String filePath, String id) { public void ultimate(String owner, String uri, String operation, String size, String filePath, String id) {
setEndTime(); logger.info("set accounting properties: owner "+owner+" uri: "+uri+" operation: "+operation+" size: "+size+ " remotePath: "+filePath+" id: "+id);
logger.info("set accounting properties: owner "+owner+" uri: "+uri+" operation: "+operation+" size: "+size);
//specific properties TODO //specific properties TODO
if(owner != null) ur.setResourceOwner(owner); if(owner != null) ur.setResourceOwner(owner);
if(uri != null) ur.setResourceSpecificProperty("objectURI", uri); if(uri != null) ur.setResourceSpecificProperty("objectURI", uri);
@ -63,24 +76,25 @@ import org.slf4j.LoggerFactory;
if(size!= null) ur.setResourceSpecificProperty("dataVolume", size); if(size!= null) ur.setResourceSpecificProperty("dataVolume", size);
if(filePath != null) ur.setResourceSpecificProperty("remotePath", filePath); if(filePath != null) ur.setResourceSpecificProperty("remotePath", filePath);
if(id!= null) ur.setResourceSpecificProperty("id", id); if(id!= null) ur.setResourceSpecificProperty("id", id);
setEndTime();
} }
@Override @Override
public void send() { public void send() {
logger.info("report sending "); logger.info("report sending...");
raFactory.sendAccountingMessage(ur); raFactory.sendAccountingMessage(ur);
} }
// private void setIpAddress() { @Deprecated
// String address=null; private void setIpAddress() {
// try { String address=null;
// address=InetAddress.getLocalHost().toString(); try {
// } catch (UnknownHostException e) { address=InetAddress.getLocalHost().toString();
// } catch (UnknownHostException e) {
// }
// logger.info("caller ip: "+address); }
// if (address!=null) ur.setResourceSpecificProperty("callerIP", address);; logger.info("caller ip: "+address);
// } if (address!=null) ur.setResourceSpecificProperty("callerIP", address);;
}
private void setStartTime() { private void setStartTime() {
//set start time //set start time
@ -98,6 +112,9 @@ import org.slf4j.LoggerFactory;
// set end time // set end time
Calendar endTime = new GregorianCalendar(); Calendar endTime = new GregorianCalendar();
Date time=endTime.getTime(); Date time=endTime.getTime();
SimpleDateFormat sdf=new SimpleDateFormat();
sdf.applyPattern("dd MM yyyy 'at' hh:mm:ss z");//format(time);
sdf.format(time);
logger.info("set end time: "+time); logger.info("set end time: "+time);
try { try {
ur.setEndTime(time); ur.setEndTime(time);

6
src/main/java/org/gcube/contentmanager/storageserver/data/ReadingMongoOplog.java
View File

@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
public class ReadingMongoOplog extends Thread{ public class ReadingMongoOplog extends Thread{
final static Logger logger=LoggerFactory.getLogger(ReadingMongoOplog.class); final static Logger logger=LoggerFactory.getLogger(ReadingMongoOplog.class);
final static String DBNAME="remotefs";
private ServerAddress[] server; private ServerAddress[] server;
private MongoClient mongoClient; private MongoClient mongoClient;
private DB local; private DB local;
@ -73,8 +74,11 @@ public class ReadingMongoOplog extends Thread{
cursor.addOption(Bytes.QUERYOPTION_AWAITDATA); cursor.addOption(Bytes.QUERYOPTION_AWAITDATA);
while (cursor.hasNext()) { while (cursor.hasNext()) {
DBObject x = cursor.next(); DBObject x = cursor.next();
logger.info("oplog current object: "+x);
ts = (BSONTimestamp) x.get("ts"); ts = (BSONTimestamp) x.get("ts");
if(x.get("o2")!=null){ String ns=(String)x.get("ns");
// check if discard or process the current DB record
if((x.get("o2")!=null) || (ns.equalsIgnoreCase(DBNAME+".fs.files"))){
if(x.containsField("o")){ if(x.containsField("o")){
// parser.jsonRecordParser(x); // parser.jsonRecordParser(x);
c.put(x); c.put(x);

34
src/main/java/org/gcube/contentmanager/storageserver/parse/JsonParser.java
View File

@ -17,6 +17,7 @@ public class JsonParser extends Thread{
private CubbyHole c; private CubbyHole c;
private int number; private int number;
private Report report; private Report report;
private String previousInsert;
public JsonParser(CubbyHole c, int number){ public JsonParser(CubbyHole c, int number){
this.c=c; this.c=c;
this.number=number; this.number=number;
@ -43,29 +44,42 @@ public class JsonParser extends Thread{
String type=(String) obj.get("type"); String type=(String) obj.get("type");
String name=(String) obj.get("name"); String name=(String) obj.get("name");
String owner=(String) obj.get("owner"); String owner=(String) obj.get("owner");
String creationTime=(String) obj.get("creationTime");
String delete=null;
logger.info("obj.get(OnDeleting) "+obj.get("onDeleting"));
if(obj.get("onDeleting") != null) delete=(String)obj.get("onDeleting");
logger.info("delete field: "+delete);
ObjectId objectId=(ObjectId)obj.get("_id"); ObjectId objectId=(ObjectId)obj.get("_id");
String id = objectId.toString(); String id = objectId.toString();
long length=(long) obj.get("length"); long length=-1;
if(obj.get("length")!=null) length=(long)obj.get("length");
logger.debug("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+ " id: "+id); logger.debug("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+ " id: "+id);
if((length >0) && (name!=null)){ if((length >0) && (filename!=null)){
//call to the accounting library //call to the accounting library
String scope=retrieveScopeFromFilename(filename); String scope=retrieveScopeFromFilename(filename);
report.init(owner, scope); report.init(owner, scope, creationTime);
report.timeUpdate(); // report.timeUpdate();
String operation=mappingOperationField(op); String operation=mappingOperationField(op, id, delete);
report.ultimate(owner, null, operation, length+"", filename, id); report.ultimate(owner, null, operation, length+"", filename, id);
report.send(); report.send();
logger.debug("[accountingCall] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner); logger.debug("[accountingCall] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner);
// }else if(op.equals("i")){
// previousInsert=id;
}else{ }else{
logger.debug("operation is not accounted"); logger.info("operation is not accounted");
} }
} }
} }
private String mappingOperationField(String op) { private String mappingOperationField(String op, String id, String onDeleting) {
logger.info("delete field: "+onDeleting);
if(op.equals("u")){ if(op.equals("u")){
return "UPDATE"; if((onDeleting != null) && (onDeleting.equals("true"))){
logger.info("found delete field");
return "DELETE";
}else
return "UPDATE";
}else if(op.equals("i")){ }else if(op.equals("i")){
return "INSERT"; return "INSERT";
}else if(op.equals("d")){ }else if(op.equals("d")){
@ -81,10 +95,6 @@ public class JsonParser extends Thread{
int i=1; int i=1;
if(split[1].equals("VOLATILE")){ if(split[1].equals("VOLATILE")){
i=2; i=2;
// scope="/"+split[1];
// if(split[2] != null && (!split[2].equals("home")) && (!split[2].equals("public"))){
// scope=scope+"/"+split[2];
// }
} }
scope="/"+split[i]; scope="/"+split[i];
i++; i++;