2017-06-23 15:56:17 +02:00
package org.gcube.accounting.analytics.persistence.couchbase ;
2017-09-04 15:43:32 +02:00
import java.text.DateFormat ;
2017-06-23 15:56:17 +02:00
import java.util.Calendar ;
import java.util.concurrent.TimeUnit ;
2017-07-31 15:39:45 +02:00
import org.gcube.accounting.aggregator.aggregation.AggregationType ;
2017-09-04 15:43:32 +02:00
import org.gcube.accounting.aggregator.utility.Utility ;
2017-06-23 15:56:17 +02:00
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration ;
import org.gcube.testutility.ScopedTest ;
import org.junit.Test ;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
import com.couchbase.client.java.Bucket ;
import com.couchbase.client.java.PersistTo ;
import com.couchbase.client.java.document.JsonDocument ;
import com.couchbase.client.java.document.json.JsonArray ;
import com.couchbase.client.java.view.ViewQuery ;
import com.couchbase.client.java.view.ViewResult ;
import com.couchbase.client.java.view.ViewRow ;
public class RemoveOldRecords extends ScopedTest {
private static final Logger logger = LoggerFactory . getLogger ( RemoveOldRecords . class ) ;
protected static JsonArray generateKey ( String scope , String key ) {
JsonArray generateKey = JsonArray . create ( ) ;
if ( scope ! = null ) {
generateKey . add ( scope ) ;
}
2017-07-31 15:39:45 +02:00
for ( String value : key . split ( " / " ) ) {
2017-06-23 15:56:17 +02:00
if ( ! value . toString ( ) . isEmpty ( ) )
generateKey . add ( Integer . parseInt ( value ) ) ;
}
return generateKey ;
}
@Test
public void removeOldRecords ( ) throws Exception {
AccountingPersistenceBackendQueryConfiguration configuration = new
AccountingPersistenceBackendQueryConfiguration ( AccountingPersistenceQueryCouchBase . class ) ;
AccountingPersistenceQueryCouchBase accountingPersistenceQueryCouchBase = new AccountingPersistenceQueryCouchBase ( ) ;
accountingPersistenceQueryCouchBase . prepareConnection ( configuration ) ;
2017-09-04 15:43:32 +02:00
DateFormat format = Utility . getUTCDateFormat ( AggregationType . DAILY . getDateFormatPattern ( ) ) ;
2017-06-23 15:56:17 +02:00
Calendar start = Calendar . getInstance ( ) ;
2017-07-31 15:39:45 +02:00
start . set ( Calendar . YEAR , 1970 ) ;
start . set ( Calendar . MONTH , Calendar . JANUARY ) ;
start . set ( Calendar . DAY_OF_MONTH , 1 ) ;
start . set ( Calendar . HOUR_OF_DAY , 0 ) ;
start . set ( Calendar . MINUTE , 0 ) ;
start . set ( Calendar . SECOND , 0 ) ;
start . set ( Calendar . MILLISECOND , 0 ) ;
2017-06-23 15:56:17 +02:00
Calendar end = Calendar . getInstance ( ) ;
end . add ( Calendar . DAY_OF_MONTH , - 60 ) ;
String startKeyString = format . format ( start . getTime ( ) ) ;
String endKeyString = format . format ( end . getTime ( ) ) ;
JsonArray startKey = generateKey ( null , startKeyString ) ;
JsonArray endKey = generateKey ( null , endKeyString ) ;
2017-07-31 15:39:45 +02:00
String designDocId = " ServiceUsageRecordAggregated " ;
2017-06-23 15:56:17 +02:00
String viewName = " all " ;
ViewQuery query = ViewQuery . from ( designDocId , viewName ) ;
query . startKey ( startKey ) ;
query . endKey ( endKey ) ;
query . reduce ( false ) ;
query . inclusiveEnd ( true ) ;
logger . debug ( " --View Query: startKey:{} - endKey:{} designDocId:{} - viewName:{} " , startKey , endKey , designDocId , viewName ) ;
Bucket [ ] buckets = new Bucket [ ] {
accountingPersistenceQueryCouchBase . bucketService ,
accountingPersistenceQueryCouchBase . bucketStorage
} ;
for ( Bucket bucket : buckets ) {
ViewResult viewResult = bucket . query ( query ) ;
int total = 0 ;
int successfullyRemoved = 0 ;
try {
for ( ViewRow row : viewResult ) {
total + + ;
try {
JsonDocument jsonDocument = row . document ( ) ;
try {
logger . trace ( " {} " , jsonDocument ) ;
String id = ( String ) jsonDocument . content ( ) . get ( " id " ) ;
bucket . remove ( id , PersistTo . MASTER , Constant . CONNECTION_TIMEOUT_BUCKET , TimeUnit . SECONDS ) ;
successfullyRemoved + + ;
} catch ( Throwable e ) {
logger . error ( " Document {} was not removed " , jsonDocument , e ) ;
}
} catch ( Throwable e ) {
logger . error ( " Error getting document from row {}. It was not removed " , row , e ) ;
}
}
} catch ( Throwable t ) {
logger . info ( " Total Number of elaborated Document to remove were {}. Successfully removed {}. Difference {} " , total , successfullyRemoved , total - successfullyRemoved ) ;
throw t ;
}
}
}
}