2014-10-22 12:38:13 +02:00
package org.gcube.contentmanager.storageserver.consumer ;
2014-03-13 15:22:33 +01:00
2021-05-12 07:10:02 +02:00
import java.io.IOException ;
import java.net.MalformedURLException ;
import java.net.URL ;
2014-10-10 16:32:23 +02:00
import java.util.List ;
2015-07-29 15:02:19 +02:00
import org.bson.types.ObjectId ;
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord ;
2021-07-15 15:20:51 +02:00
import org.gcube.common.authorization.library.provider.AccessTokenProvider ;
2015-07-29 15:02:19 +02:00
import org.gcube.common.scope.api.ScopeProvider ;
2014-03-13 15:22:33 +01:00
import org.gcube.contentmanager.storageserver.accounting.Report ;
import org.gcube.contentmanager.storageserver.accounting.ReportConfig ;
import org.gcube.contentmanager.storageserver.accounting.ReportException ;
import org.gcube.contentmanager.storageserver.accounting.ReportFactory ;
import org.gcube.contentmanager.storageserver.data.CubbyHole ;
2015-07-29 15:02:19 +02:00
import org.gcube.contentmanager.storageserver.data.OpLogRemoteObject ;
2014-04-16 10:54:47 +02:00
import org.gcube.contentmanager.storageserver.parse.utils.ValidationUtils ;
2014-06-05 19:27:16 +02:00
import org.gcube.contentmanager.storageserver.store.MongoDB ;
2014-10-22 17:47:24 +02:00
import org.gcube.contentmanager.storageserver.store.StorageStatusObject ;
2021-05-12 07:10:02 +02:00
import org.gcube.contentmanager.storageserver.utils.Utils ;
import org.gcube.oidc.rest.JWTToken ;
import org.gcube.oidc.rest.OpenIdConnectRESTHelper ;
import org.gcube.oidc.rest.OpenIdConnectRESTHelperException ;
2014-03-13 15:22:33 +01:00
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
import com.mongodb.DBObject ;
2014-10-22 12:38:13 +02:00
public class UserAccountingConsumer extends Thread {
2014-03-13 15:22:33 +01:00
2014-10-22 12:38:13 +02:00
final static Logger logger = LoggerFactory . getLogger ( UserAccountingConsumer . class ) ;
2014-03-21 16:50:43 +01:00
final static int MINUTE_DECREMENT = - 2 ;
2014-03-13 15:22:33 +01:00
private CubbyHole c ;
private int number ;
private Report report ;
2014-04-01 11:20:28 +02:00
// object fields
private String op ;
2014-06-05 19:27:16 +02:00
private String user ;
private String password ;
2021-05-12 07:10:02 +02:00
private String clientId ;
private String secret ;
private String oidcEndpoint ;
2014-06-05 19:27:16 +02:00
String [ ] server ;
2014-10-10 16:32:23 +02:00
List < String > dtsHosts ;
2015-07-29 15:02:19 +02:00
// private String id;
2014-06-05 19:27:16 +02:00
2021-05-12 07:10:02 +02:00
public UserAccountingConsumer ( String [ ] srvs , CubbyHole c , int number , List < String > dtsHosts , String clientId , String secret , String authEndpoint ) {
2014-03-13 15:22:33 +01:00
this . c = c ;
this . number = number ;
2014-06-05 19:27:16 +02:00
this . server = srvs ;
2014-10-10 16:32:23 +02:00
this . dtsHosts = dtsHosts ;
2021-05-12 07:10:02 +02:00
this . clientId = clientId ;
this . secret = secret ;
this . oidcEndpoint = authEndpoint ;
2014-06-05 19:27:16 +02:00
// init the accounting report
try {
init ( ) ;
} catch ( ReportException e ) {
throw new RuntimeException ( " Accounting report Exception initialization " ) ;
}
}
2021-05-12 07:10:02 +02:00
public UserAccountingConsumer ( String [ ] srvs , String user , String password , CubbyHole c , int number , List < String > dtsHosts , String clientId , String secret , String authEndpoint ) {
2014-06-05 19:27:16 +02:00
this . c = c ;
this . number = number ;
this . server = srvs ;
2014-10-10 16:32:23 +02:00
this . dtsHosts = dtsHosts ;
2014-06-05 19:27:16 +02:00
this . user = user ;
this . password = password ;
2021-05-12 07:10:02 +02:00
this . clientId = clientId ;
this . secret = secret ;
this . oidcEndpoint = authEndpoint ;
2014-03-19 18:08:15 +01:00
// init the accounting report
2014-03-13 15:22:33 +01:00
try {
init ( ) ;
} catch ( ReportException e ) {
throw new RuntimeException ( " Accounting report Exception initialization " ) ;
}
}
2014-09-10 10:23:30 +02:00
2014-03-13 15:22:33 +01:00
private void init ( ) throws ReportException {
2014-09-10 10:23:30 +02:00
report = ReportFactory . getReport ( ReportConfig . ACCOUNTING_TYPE ) ;
2014-04-15 17:57:57 +02:00
report . init ( ) ;
2014-03-13 15:22:33 +01:00
}
public void run ( ) {
2016-11-23 18:36:31 +01:00
logger . debug ( " Consuming SU started " ) ;
MongoDB mongo = null ;
2014-03-13 15:22:33 +01:00
while ( true ) {
2016-11-23 18:36:31 +01:00
DBObject x = null ;
2014-06-09 10:10:13 +02:00
try {
2016-11-23 18:36:31 +01:00
logger . debug ( " SU waiting next record... " ) ;
2014-06-09 10:10:13 +02:00
x = c . get ( ) ;
2016-11-23 18:36:31 +01:00
logger . debug ( " Consumer # " + this . number + " got: " + x ) ;
2016-11-17 11:12:36 +01:00
//get operation
2014-06-09 10:10:13 +02:00
op = ( String ) x . get ( " op " ) ;
// retrieve object fields
DBObject obj = ( DBObject ) x . get ( " o " ) ;
2015-07-29 15:02:19 +02:00
OpLogRemoteObject record = retrieveObjectFields ( obj ) ;
2014-09-10 10:23:30 +02:00
// set object dimension
2015-07-29 15:02:19 +02:00
2016-11-23 18:36:31 +01:00
logger . debug ( " [recordCheck] operation: " + op + " name: " + record . getName ( ) + " type: " + record . getType ( ) + " path: " + record . getFilename ( ) + " dir Path: " + record . getDir ( ) + " length: " + record . getLength ( ) + " owner: " + record . getOwner ( ) + " lastOperation " + record . getLastOperation ( ) + " \ n \ t lastUser: " + record . getLastUser ( ) + " lastAccess: " + record . getLastAccess ( ) ) ;
2015-10-09 11:32:18 +02:00
if ( ( ( record . getLength ( ) > 0 ) & & ( ( ( record . getFilename ( ) ! = null ) & & ( record . getFilename ( ) . length ( ) > 0 ) & & ( record . getDir ( ) . length ( ) > 0 ) & & ( record . getDir ( ) . contains ( " / " ) ) ) | | ( record . getLinkCount ( ) > 0 ) ) ) ) {
2016-11-23 18:36:31 +01:00
String id = ( ( DBObject ) x . get ( " o " ) ) . get ( " _id " ) . toString ( ) ;
logger . debug ( " Consumer id " + id ) ;
2014-06-09 10:10:13 +02:00
//convert from byte to kb
2015-07-29 15:02:19 +02:00
record . setLength ( record . getLength ( ) / 1024 ) ;
2014-09-10 10:23:30 +02:00
// check scope
2014-06-09 10:10:13 +02:00
String scope = null ;
2015-10-09 11:32:18 +02:00
String pathString = ( String ) obj . get ( " onScope " ) ;
2016-11-23 18:36:31 +01:00
logger . debug ( " [recordCheck] it is a link to: " + pathString ) ;
2015-10-09 11:32:18 +02:00
if ( ( record . getDir ( ) ! = null ) & & ( record . getDir ( ) . contains ( " / " ) ) ) {
scope = retrieveScopeFromRemoteFilePath ( record . getDir ( ) ) ;
} else {
2014-06-09 10:10:13 +02:00
// field added on storage manager library for retrieve scope. Used only if it is a link delete
scope = retrieveScopeFromRemoteFilePath ( pathString ) ;
2014-04-16 10:54:47 +02:00
}
2016-11-23 18:36:31 +01:00
logger . debug ( " scope retrieved " ) ;
2014-06-09 10:10:13 +02:00
boolean validScope = ValidationUtils . validationScope ( scope ) ;
if ( validScope ) {
2015-07-29 15:02:19 +02:00
if ( record . getDelete ( ) ! = null ) {
record . setLastOperation ( " DELETE " ) ;
2016-11-23 18:36:31 +01:00
} else if ( ( record . getLastOperation ( ) ! = null ) & & ( op ! = null ) & & ( record . getLastOperation ( ) . equalsIgnoreCase ( " LINK " ) ) & & ( op . equalsIgnoreCase ( " u " ) ) ) { //add check on operation
2014-06-09 10:10:13 +02:00
// it is an update on a link object this operation doesn't be accounted
2018-07-16 16:00:24 +02:00
logger . debug ( " [recordCheck] update on link object is not accounted. Skip next " ) ;
2014-06-09 10:10:13 +02:00
continue ;
2015-07-29 15:02:19 +02:00
} else if ( ( record . getLastOperation ( ) = = null ) | | ( record . getLastUser ( ) = = null ) ) {
logger . warn ( " lastOperation: " + record . getLastOperation ( ) + " lastUser: " + record . getLastUser ( ) + " . These values cannot be null. Skip next " ) ;
2014-10-22 12:38:13 +02:00
continue ;
} else {
// if the lastoperation is download and the caller contains a dts host then it isn't a real user but it is a workspace accounting record
2015-07-29 15:02:19 +02:00
if ( ( dtsHosts ! = null ) & & ( record . getLastOperation ( ) ! = null ) & & ( record . getLastOperation ( ) . equalsIgnoreCase ( " DOWNLOAD " ) ) ) {
2016-11-17 11:12:36 +01:00
logger . debug ( " [recordCheck] check if the caller is from dts. CallerIP: " + record . getCallerIp ( ) + " dts value: " + dtsHosts ) ;
2014-10-22 12:38:13 +02:00
for ( String host : dtsHosts ) {
logger . debug ( " scan " + host ) ;
2015-07-29 15:02:19 +02:00
if ( record . getCallerIp ( ) . contains ( host ) ) {
record . setLastOperation ( " workspace.accounting " ) ;
2016-11-17 11:12:36 +01:00
logger . debug ( " [recordCheck] the caller is dts service: caller " + record . getCallerIp ( ) + " dts host: " + host + " the new user is: " + record . getLastUser ( ) ) ;
2014-10-22 12:38:13 +02:00
}
}
2016-11-23 18:36:31 +01:00
} else if ( ( record . getLastOperation ( ) ! = null ) & & ( record . getLastOperation ( ) . equalsIgnoreCase ( " UPLOAD " ) ) & & op . equalsIgnoreCase ( " u " ) ) {
//it is only a metadata update on a previous insert so it is not need to send the record
logger . info ( " [recordCheck] metadata update on a previous insert. Record discarded: " + id ) ;
continue ;
2014-10-22 12:38:13 +02:00
}
2014-10-10 16:32:23 +02:00
}
2016-11-23 18:36:31 +01:00
logger . debug ( " [recordCheck] accounting operation " + record . getLastOperation ( ) ) ;
2014-10-22 17:47:24 +02:00
StorageStatusObject ssr = null ;
2015-07-29 15:02:19 +02:00
if ( isNeedSSReport ( record . getLastOperation ( ) ) ) {
2014-10-07 17:06:09 +02:00
try {
2016-11-23 18:36:31 +01:00
logger . debug ( " [recordCheck] update SS record yet " ) ;
if ( mongo = = null )
mongo = new MongoDB ( server , user , password ) ;
2018-06-20 16:46:42 +02:00
if ( record . getLastOperation ( ) . equalsIgnoreCase ( " COPY " ) | | record . getLastOperation ( ) . equalsIgnoreCase ( " SOFT_COPY " ) )
2015-07-29 15:02:19 +02:00
record . setOwner ( record . getLastUser ( ) ) ;
ssr = new StorageStatusObject ( record . getOwner ( ) , record . getLength ( ) , 1 ) ;
ssr = mongo . updateUserVolume ( ssr , record . getLastOperation ( ) ) ;
2018-07-16 16:00:24 +02:00
// mongo.close();
2014-10-07 17:06:09 +02:00
} catch ( Exception e ) {
2018-07-16 16:00:24 +02:00
e . printStackTrace ( ) ;
2015-10-14 11:48:30 +02:00
mongo . close ( ) ;
2018-07-16 16:00:24 +02:00
logger . error ( " [recordCheck] Problem when updating storage status record: " + e . getMessage ( ) ) ;
2014-10-07 17:06:09 +02:00
}
2014-09-10 10:23:30 +02:00
}
try {
2015-07-29 15:02:19 +02:00
// scope set add in 1.3 for new accounting libs
ScopeProvider . instance . set ( scope ) ;
2014-09-10 10:23:30 +02:00
//call to the accounting library
2014-10-07 17:06:09 +02:00
if ( ssr ! = null )
2015-07-29 15:02:19 +02:00
report ( record , scope , ssr . getVolume ( ) + " " , ssr . getCount ( ) + " " ) ;
2014-10-07 17:06:09 +02:00
else
2015-07-29 15:02:19 +02:00
report ( record , scope , null , null ) ;
2014-09-10 10:23:30 +02:00
} catch ( Exception e ) {
2016-11-17 11:12:36 +01:00
logger . error ( " [recordCheck] Problem sending accounting report. Exception message: " + e . getMessage ( ) ) ;
2014-06-09 10:10:13 +02:00
}
} else {
2018-07-16 16:00:24 +02:00
logger . warn ( " [recordCheck] operation " + record . getLastOperation ( ) + " is not accounted: invalid scope: " + scope ) ;
2014-06-09 10:10:13 +02:00
}
} else {
2018-07-16 16:00:24 +02:00
logger . debug ( " operation " + record . getLastOperation ( ) + " is not accounted " ) ;
2014-06-09 10:10:13 +02:00
}
} catch ( Exception e ) {
2016-11-11 12:03:52 +01:00
logger . error ( " ERROR Processing record: " + x + " Exception throws: " + e . getMessage ( ) ) ;
2014-06-09 10:10:13 +02:00
logger . info ( " skip to next record " ) ;
2014-06-30 14:42:50 +02:00
if ( mongo ! = null )
mongo . close ( ) ;
2016-11-23 18:36:31 +01:00
2014-06-09 10:10:13 +02:00
}
2014-03-13 15:22:33 +01:00
}
}
2014-09-10 10:23:30 +02:00
2014-10-22 12:38:13 +02:00
/ * *
* check if the lastOperation is a valid operation for storage status information record
* @param lastOperation
* @return
* /
2014-10-07 17:06:09 +02:00
private boolean isNeedSSReport ( String lastOperation ) {
2016-11-23 18:36:31 +01:00
logger . trace ( " Last operation is " + lastOperation ) ;
2018-06-20 16:46:42 +02:00
if ( lastOperation . equalsIgnoreCase ( " UPLOAD " ) | | lastOperation . equalsIgnoreCase ( " COPY " ) | | lastOperation . equalsIgnoreCase ( " SOFT_COPY " ) | | lastOperation . equalsIgnoreCase ( " DELETE " ) )
2014-10-07 17:06:09 +02:00
return true ;
return false ;
}
2014-10-22 12:38:13 +02:00
2014-10-07 17:06:09 +02:00
2014-09-10 10:23:30 +02:00
public void runWithoutThread ( DBObject x ) {
try {
2014-10-10 16:32:23 +02:00
report = ReportFactory . getReport ( ReportConfig . ACCOUNTING_TYPE ) ;
2014-09-10 10:23:30 +02:00
} catch ( ReportException e1 ) {
// TODO Auto-generated catch block
e1 . printStackTrace ( ) ;
}
report . init ( ) ;
2014-10-13 12:24:19 +02:00
run ( ) ;
2014-09-10 10:23:30 +02:00
}
2014-03-13 15:22:33 +01:00
2015-07-29 15:02:19 +02:00
private OpLogRemoteObject retrieveObjectFields ( DBObject obj ) {
OpLogRemoteObject record = new OpLogRemoteObject ( ) ;
record . setFilename ( ( String ) obj . get ( " filename " ) ) ;
record . setType ( ( String ) obj . get ( " type " ) ) ;
record . setName ( ( String ) obj . get ( " name " ) ) ;
record . setOwner ( ( String ) obj . get ( " owner " ) ) ;
record . setCreationTime ( ( String ) obj . get ( " creationTime " ) ) ;
2015-10-09 11:32:18 +02:00
record . setDir ( ( String ) obj . get ( " dir " ) ) ;
2015-07-29 15:02:19 +02:00
if ( obj . get ( " lastAccess " ) ! = null ) record . setLastAccess ( ( String ) obj . get ( " lastAccess " ) ) ;
if ( obj . get ( " callerIP " ) ! = null ) record . setCallerIp ( ( String ) obj . get ( " callerIP " ) ) ;
if ( obj . get ( " lastOperation " ) ! = null ) record . setLastOperation ( ( String ) obj . get ( " lastOperation " ) ) ;
if ( obj . get ( " lastUser " ) ! = null ) record . setLastUser ( ( String ) obj . get ( " lastUser " ) ) ;
if ( obj . get ( " linkCount " ) ! = null ) record . setLinkCount ( ( int ) obj . get ( " linkCount " ) ) ;
if ( obj . get ( " onDeleting " ) ! = null ) record . setDelete ( ( String ) obj . get ( " onDeleting " ) ) ;
if ( obj . get ( " _id " ) ! = null ) {
ObjectId id = ( ObjectId ) obj . get ( " _id " ) ;
if ( ObjectId . isValid ( id . toString ( ) ) )
record . setId ( id . toString ( ) ) ;
}
long length = - 1 ;
if ( obj . get ( " length " ) ! = null ) record . setLength ( ( long ) obj . get ( " length " ) ) ;
return record ;
2014-03-19 18:08:15 +01:00
}
2021-05-12 17:11:18 +02:00
private void report ( OpLogRemoteObject record , String scope , String totVolume , String totCount ) {
2021-05-12 07:10:02 +02:00
2014-06-05 19:27:16 +02:00
// ACCOUNTING CALL TYPE: STORAGE USAGE
2015-10-14 11:48:30 +02:00
StorageUsageRecord sur = report . setGenericProperties ( null , " storage-usage " , record . getLastUser ( ) , scope , record . getCreationTime ( ) , record . getLastAccess ( ) , record . getOwner ( ) , record . getLastOperation ( ) , record . getLength ( ) + " " ) ;
2015-07-29 15:02:19 +02:00
sur = report . setSpecificProperties ( sur , record . getFilename ( ) , " STORAGE " , record . getCallerIp ( ) , record . getId ( ) ) ;
2021-05-12 17:11:18 +02:00
logger . info ( " storage usage record ready " ) ;
logger . debug ( " try to call: " + oidcEndpoint ) ;
URL oidcAddress ;
try {
logger . trace ( " preparing URL " ) ;
oidcAddress = new URL ( oidcEndpoint ) ;
logger . trace ( " URL ready: " + oidcAddress ) ;
Utils . setToken ( oidcAddress , clientId , secret ) ;
logger . trace ( " token ready " ) ;
} catch ( MalformedURLException e ) {
// TODO Auto-generated catch block
e . printStackTrace ( ) ;
logger . error ( " Malformed URL " , e ) ;
} catch ( OpenIdConnectRESTHelperException e ) {
// TODO Auto-generated catch block
e . printStackTrace ( ) ;
logger . error ( " OpenIdConnectRESTHelperException " , e ) ;
}
2021-05-12 07:10:02 +02:00
logger . debug ( " sending record to the accounting " ) ;
2014-10-07 17:06:09 +02:00
report . printRecord ( sur ) ;
2014-06-05 19:27:16 +02:00
report . send ( sur ) ;
2014-03-13 15:22:33 +01:00
}
2014-10-07 17:06:09 +02:00
2014-03-13 15:22:33 +01:00
2014-04-01 11:20:28 +02:00
2014-03-19 18:08:15 +01:00
private String retrieveScopeFromRemoteFilePath ( String filename ) {
2014-03-13 15:22:33 +01:00
String [ ] split = filename . split ( " / " ) ;
if ( split . length > 0 ) {
String scope = null ;
int i = 1 ;
if ( split [ 1 ] . equals ( " VOLATILE " ) ) {
i = 2 ;
}
scope = " / " + split [ i ] ;
i + + ;
while ( ( ! split [ i ] . equals ( " home " ) ) & & ( ! split [ i ] . equals ( " public " ) ) ) {
scope = scope + " / " + split [ i ] ;
2016-11-23 18:36:31 +01:00
logger . debug ( " scope building: " + scope ) ;
2014-03-13 15:22:33 +01:00
i + + ;
}
2016-11-23 18:36:31 +01:00
logger . debug ( " [recordCheck] scope retrieved: " + scope ) ;
2014-03-13 15:22:33 +01:00
return scope ;
} else logger . error ( " Scope bad format: scope not retrieved from string: " + filename ) ;
return null ;
2021-05-12 17:11:18 +02:00
}
2014-10-22 12:38:13 +02:00
2014-03-13 15:22:33 +01:00
}