git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/storage-manager-trigger@97114 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
dae808dce6
commit
9387a59f8f
|
@ -76,48 +76,56 @@ public class JsonParser extends Thread{
|
||||||
|
|
||||||
public void run() {
|
public void run() {
|
||||||
while(true){
|
while(true){
|
||||||
DBObject x=c.get();
|
DBObject x=null;
|
||||||
logger.info("Consumer #" + this.number + " got: " + x);
|
try{
|
||||||
op = (String) x.get("op");
|
x=c.get();
|
||||||
// retrieve object fields
|
logger.info("Consumer #" + this.number + " got: " + x);
|
||||||
DBObject obj=(DBObject)x.get("o");
|
op = (String) x.get("op");
|
||||||
retrieveObjectFields(obj);
|
// retrieve object fields
|
||||||
long length=-1;
|
DBObject obj=(DBObject)x.get("o");
|
||||||
if(obj.get("length")!=null) length=(long)obj.get("length");
|
retrieveObjectFields(obj);
|
||||||
logger.info("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+"\n\t cretionTime: "+creationTime+ " lastOperation "+lastOperation+" lastUser: "+lastUser+" lastAccess: "+lastAccess);
|
long length=-1;
|
||||||
if(((length >0) && (((filename!=null) && (filename.contains("/"))) || (linkCount > 0)))){
|
if(obj.get("length")!=null) length=(long)obj.get("length");
|
||||||
//convert from byte to kb
|
logger.info("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+"\n\t cretionTime: "+creationTime+ " lastOperation "+lastOperation+" lastUser: "+lastUser+" lastAccess: "+lastAccess);
|
||||||
length=length/1024;
|
if(((length >0) && (((filename!=null) && (filename.contains("/"))) || (linkCount > 0)))){
|
||||||
String scope=null;
|
//convert from byte to kb
|
||||||
if((filename!=null)&& (filename.contains("/")))
|
length=length/1024;
|
||||||
scope=retrieveScopeFromRemoteFilePath(filename);
|
String scope=null;
|
||||||
else{
|
if((filename!=null)&& (filename.contains("/")))
|
||||||
// field added on storage manager library for retrieve scope. Used only if it is a link delete
|
scope=retrieveScopeFromRemoteFilePath(filename);
|
||||||
String pathString=(String)obj.get("onScope");
|
else{
|
||||||
scope=retrieveScopeFromRemoteFilePath(pathString);
|
// field added on storage manager library for retrieve scope. Used only if it is a link delete
|
||||||
}
|
String pathString=(String)obj.get("onScope");
|
||||||
boolean validScope=ValidationUtils.validationScope(scope);
|
scope=retrieveScopeFromRemoteFilePath(pathString);
|
||||||
if(validScope){
|
|
||||||
if(delete!=null){
|
|
||||||
lastOperation="DELETE";
|
|
||||||
}else if ((lastOperation != null) && (op != null) && (lastOperation.equalsIgnoreCase("LINK")) && (op.equalsIgnoreCase("u"))){
|
|
||||||
// it is an update on a link object this operation doesn't be accounted
|
|
||||||
logger.info("update on link object is not accounted. Skip next ");
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
// operation=mappingOperationField(op, id, delete, lastAccess);
|
boolean validScope=ValidationUtils.validationScope(scope);
|
||||||
MongoDB mongo=new MongoDB(server, user, password);
|
if(validScope){
|
||||||
StorageStatusRecord ssr=mongo.update(lastUser, length, 1, lastOperation);
|
if(delete!=null){
|
||||||
//call to the accounting library
|
lastOperation="DELETE";
|
||||||
report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, ssr.getVolume()+"", ssr.getCount()+"");
|
}else if ((lastOperation != null) && (op != null) && (lastOperation.equalsIgnoreCase("LINK")) && (op.equalsIgnoreCase("u"))){
|
||||||
logger.info(" operation accounted "+lastOperation);
|
// it is an update on a link object this operation doesn't be accounted
|
||||||
logger.info("\n[accountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp);
|
logger.info("update on link object is not accounted. Skip next ");
|
||||||
}else{
|
continue;
|
||||||
logger.info("operation is not accounted: invalid scope: "+scope);
|
}
|
||||||
}
|
// operation=mappingOperationField(op, id, delete, lastAccess);
|
||||||
}else{
|
MongoDB mongo=new MongoDB(server, user, password);
|
||||||
logger.info("operation is not accounted");
|
StorageStatusRecord ssr=mongo.update(lastUser, length, 1, lastOperation);
|
||||||
}
|
mongo.close();
|
||||||
|
//call to the accounting library
|
||||||
|
report( filename, owner, creationTime, length, scope, lastOperation, callerIp, lastAccess, lastUser, ssr.getVolume()+"", ssr.getCount()+"");
|
||||||
|
logger.info(" operation accounted "+lastOperation);
|
||||||
|
logger.info("\n[accountingCall] operation: "+lastOperation+"\n\t name: "+name+"\n\t type: "+type+"\n\t path: "+filename+"\n\t length: "+length+"\n\t owner: "+owner+"\n\t cretionTime: "+creationTime+"\n\t scope: "+scope+"\n\t lastOperation "+lastOperation+"\n\t lastUser: "+lastUser+"\n\t lastAccess: "+lastAccess+"\n\t callerIp: "+callerIp);
|
||||||
|
}else{
|
||||||
|
logger.info("operation is not accounted: invalid scope: "+scope);
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
logger.info("operation is not accounted");
|
||||||
|
}
|
||||||
|
|
||||||
|
}catch(Exception e){
|
||||||
|
logger.error("ERROR Processing record: "+x+" Exception throws: "+e.getMessage());
|
||||||
|
logger.info("skip to next record ");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -49,65 +49,17 @@ public class MongoDB {
|
||||||
this.pwd=password;
|
this.pwd=password;
|
||||||
this.user=user;
|
this.user=user;
|
||||||
this.collection=DEFAULT_COLLECTION;
|
this.collection=DEFAULT_COLLECTION;
|
||||||
db=getDB();
|
// db=getDB();
|
||||||
ssCollection = db.getCollection(collection);
|
// ssCollection = db.getCollection(collection);
|
||||||
}
|
}
|
||||||
|
|
||||||
private DB getDB() {
|
|
||||||
if(db != null){
|
|
||||||
// check if the old server is primary
|
|
||||||
try{
|
|
||||||
DB db = mongo.getDB(ACCOUNTING_DB);
|
|
||||||
}catch(Exception e ){
|
|
||||||
logger.warn("the server now is not a primary ");
|
|
||||||
db=null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if(db==null){
|
|
||||||
int i=-1;
|
|
||||||
for(String srv : server){
|
|
||||||
try {
|
|
||||||
i++;
|
|
||||||
if(mongo!=null)
|
|
||||||
mongo.close();
|
|
||||||
MongoOptions options=new MongoOptions();
|
|
||||||
options.autoConnectRetry=true;
|
|
||||||
options.socketKeepAlive=true;
|
|
||||||
options.maxWaitTime=240000;
|
|
||||||
options.connectionsPerHost=35;
|
|
||||||
mongo = new Mongo(srv, options);
|
|
||||||
// mongo.getMongoOptions().autoConnectRetry=true;
|
|
||||||
// mongo.getMongoOptions().socketKeepAlive=true;
|
|
||||||
logger.debug("Istantiate MongoDB with options: "+mongo.getMongoOptions());
|
|
||||||
db = mongo.getDB(ACCOUNTING_DB);
|
|
||||||
// check on user and password for non authenticate mode
|
|
||||||
if(user==null) user="";
|
|
||||||
if(pwd==null) pwd="";
|
|
||||||
boolean auth = db.authenticate(user, pwd.toCharArray());
|
|
||||||
if(auth) logger.info("mongo is in authenticate mode");
|
|
||||||
else logger.info("mongo is not in authenticate mode");
|
|
||||||
ssCollection = db.getCollection(collection);
|
|
||||||
ssCollection.findOne();
|
|
||||||
// GridFS gfs = new GridFS(db);
|
|
||||||
String firstServer = server[0];
|
|
||||||
server[0] = srv;
|
|
||||||
server[i]=firstServer;
|
|
||||||
break;
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.warn("server "+srv+" is not a primary retry ");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return db;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void put(String consumer, long volume, int count){
|
public void put(String consumer, long volume, int count){
|
||||||
BasicDBObject doc = new BasicDBObject("consumer", consumer)
|
BasicDBObject doc = new BasicDBObject("consumer", consumer)
|
||||||
.append("volume", volume)
|
.append("volume", volume)
|
||||||
.append("count", count);
|
.append("count", count);
|
||||||
ssCollection.insert(doc);
|
getCollection().insert(doc);
|
||||||
|
close();
|
||||||
}
|
}
|
||||||
|
|
||||||
public StorageStatusRecord update(String consumer, long volume, int count, String operation){
|
public StorageStatusRecord update(String consumer, long volume, int count, String operation){
|
||||||
|
@ -123,13 +75,50 @@ public class MongoDB {
|
||||||
DBObject documentBuilder = BasicDBObjectBuilder.start()
|
DBObject documentBuilder = BasicDBObjectBuilder.start()
|
||||||
.add("volume", volume).add("count", count).get();
|
.add("volume", volume).add("count", count).get();
|
||||||
// get the dbobject from builder and Inserting document
|
// get the dbobject from builder and Inserting document
|
||||||
ssCollection.update(query,new BasicDBObject("$set", documentBuilder), true, false);
|
getCollection().update(query,new BasicDBObject("$set", documentBuilder), true, false);
|
||||||
|
close();
|
||||||
}else{
|
}else{
|
||||||
put(consumer, volume, count);
|
put(consumer, volume, count);
|
||||||
}
|
}
|
||||||
return new StorageStatusRecord(consumer, volume, count);
|
return new StorageStatusRecord(consumer, volume, count);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public StorageStatusRecord get(String consumer){
|
||||||
|
BasicDBObject query = new BasicDBObject("consumer", consumer);
|
||||||
|
DBCursor cursor=getCollection().find(query);
|
||||||
|
DBObject obj=null;
|
||||||
|
try{
|
||||||
|
if(cursor.hasNext()){
|
||||||
|
obj=cursor.next();
|
||||||
|
|
||||||
|
}
|
||||||
|
}finally{
|
||||||
|
cursor.close();
|
||||||
|
}
|
||||||
|
if(obj!=null){
|
||||||
|
String cons=null;
|
||||||
|
if(obj.containsField("consumer")) cons=(String) obj.get("consumer");
|
||||||
|
else logger.error("incomplete record found. consumer field is missing");
|
||||||
|
long vol =0;
|
||||||
|
if(obj.containsField("volume")) vol=(long) obj.get("volume");
|
||||||
|
else logger.error("incomplete record found. volume field is missing");
|
||||||
|
int count=0;
|
||||||
|
if(obj.containsField("count")) count=(int) obj.get("count");
|
||||||
|
else logger.error("incomplete record found. count field is missing");
|
||||||
|
String id=(String)obj.get("id");
|
||||||
|
return new StorageStatusRecord(id, cons, vol, count, obj);
|
||||||
|
}else{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void close(){
|
||||||
|
if(mongo!=null)
|
||||||
|
mongo.close();
|
||||||
|
}
|
||||||
|
|
||||||
private long setVolume(long volume, long currentVolume, String operation) {
|
private long setVolume(long volume, long currentVolume, String operation) {
|
||||||
logger.info("accounting: operation "+operation+" total Volume "+currentVolume+" current volume "+volume);
|
logger.info("accounting: operation "+operation+" total Volume "+currentVolume+" current volume "+volume);
|
||||||
if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){
|
if(operation.equalsIgnoreCase("UPLOAD") || operation.equalsIgnoreCase("COPY")){
|
||||||
|
@ -152,33 +141,61 @@ public class MongoDB {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private DB getDB() {
|
||||||
public StorageStatusRecord get(String consumer){
|
if(db != null){
|
||||||
BasicDBObject query = new BasicDBObject("consumer", consumer);
|
// check if the old server is primary
|
||||||
DBCursor cursor=ssCollection.find(query);
|
try{
|
||||||
DBObject obj=null;
|
DB db = mongo.getDB(ACCOUNTING_DB);
|
||||||
try{
|
}catch(Exception e ){
|
||||||
if(cursor.hasNext()){
|
logger.warn("the server now is not a primary ");
|
||||||
obj=cursor.next();
|
db=null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}finally{
|
if(db==null){
|
||||||
cursor.close();
|
int i=-1;
|
||||||
}
|
for(String srv : server){
|
||||||
if(obj!=null){
|
try {
|
||||||
String cons=null;
|
i++;
|
||||||
if(obj.containsField("consumer")) cons=(String) obj.get("consumer");
|
ssCollection=null;
|
||||||
else logger.error("incomplete record found. consumer field is missing");
|
if(mongo!=null)
|
||||||
long vol =0;
|
mongo.close();
|
||||||
if(obj.containsField("volume")) vol=(long) obj.get("volume");
|
MongoOptions options=new MongoOptions();
|
||||||
else logger.error("incomplete record found. volume field is missing");
|
options.autoConnectRetry=true;
|
||||||
int count=0;
|
options.socketKeepAlive=true;
|
||||||
if(obj.containsField("count")) count=(int) obj.get("count");
|
options.maxWaitTime=240000;
|
||||||
else logger.error("incomplete record found. count field is missing");
|
options.connectionsPerHost=35;
|
||||||
String id=(String)obj.get("id");
|
mongo = new Mongo(srv, options);
|
||||||
return new StorageStatusRecord(id, cons, vol, count, obj);
|
// mongo.getMongoOptions().autoConnectRetry=true;
|
||||||
}else
|
// mongo.getMongoOptions().socketKeepAlive=true;
|
||||||
return null;
|
logger.debug("Istantiate MongoDB with options: "+mongo.getMongoOptions());
|
||||||
|
db = mongo.getDB(ACCOUNTING_DB);
|
||||||
|
// check on user and password for non authenticate mode
|
||||||
|
if(user==null) user="";
|
||||||
|
if(pwd==null) pwd="";
|
||||||
|
boolean auth = db.authenticate(user, pwd.toCharArray());
|
||||||
|
if(auth) logger.info("mongo is in authenticate mode");
|
||||||
|
else logger.info("mongo is not in authenticate mode");
|
||||||
|
if(ssCollection == null)
|
||||||
|
ssCollection=db.getCollection(collection);
|
||||||
|
ssCollection.findOne();
|
||||||
|
String firstServer = server[0];
|
||||||
|
server[0] = srv;
|
||||||
|
server[i]=firstServer;
|
||||||
|
break;
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.warn("server "+srv+" is not a primary retry ");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return db;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private DBCollection getCollection() {
|
||||||
|
if(ssCollection==null)
|
||||||
|
return getDB().getCollection(collection);
|
||||||
|
else
|
||||||
|
return ssCollection;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,10 +16,14 @@ public class MongoDBTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
public void putAndRetrieve(){
|
public void update(){
|
||||||
mongo.update("test.consumer", 100, -1, "UPLOAD");
|
mongo.update("test.consumer", 100, 1, "UPLOAD");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void put(){
|
||||||
|
mongo.put("test.consumer2", 100, 1 );
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue