121 lines
3.9 KiB
Java
121 lines
3.9 KiB
Java
package org.gcube.accounting.analytics.persistence.couchbase;
|
|
|
|
import java.text.DateFormat;
|
|
import java.util.ArrayList;
|
|
import java.util.Calendar;
|
|
import java.util.List;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import org.gcube.accounting.aggregator.aggregation.AggregationType;
|
|
import org.gcube.accounting.aggregator.utility.Utility;
|
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
|
|
import org.gcube.testutility.ScopedTest;
|
|
import org.junit.Test;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import com.couchbase.client.java.Bucket;
|
|
import com.couchbase.client.java.PersistTo;
|
|
import com.couchbase.client.java.document.JsonDocument;
|
|
import com.couchbase.client.java.document.json.JsonArray;
|
|
import com.couchbase.client.java.view.ViewQuery;
|
|
import com.couchbase.client.java.view.ViewResult;
|
|
import com.couchbase.client.java.view.ViewRow;
|
|
|
|
public class RemoveOldRecords extends ScopedTest {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(RemoveOldRecords.class);
|
|
|
|
protected static JsonArray generateKey(String scope,String key){
|
|
JsonArray generateKey = JsonArray.create();
|
|
if (scope!=null){
|
|
generateKey.add(scope);
|
|
}
|
|
for (String value: key.split("/")){
|
|
if (!value.toString().isEmpty())
|
|
generateKey.add(Integer.parseInt(value));
|
|
}
|
|
return generateKey;
|
|
|
|
}
|
|
|
|
@Test
|
|
public void removeOldRecords() throws Exception{
|
|
AccountingPersistenceBackendQueryConfiguration configuration = new
|
|
AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchBase.class);
|
|
AccountingPersistenceQueryCouchBase accountingPersistenceQueryCouchBase = new AccountingPersistenceQueryCouchBase();
|
|
accountingPersistenceQueryCouchBase.prepareConnection(configuration);
|
|
|
|
DateFormat format = Utility.getUTCDateFormat(AggregationType.DAILY.getDateFormatPattern());
|
|
|
|
Calendar start = Calendar.getInstance();
|
|
start.set(Calendar.YEAR, 1970);
|
|
start.set(Calendar.MONTH, Calendar.JANUARY);
|
|
start.set(Calendar.DAY_OF_MONTH, 1);
|
|
start.set(Calendar.HOUR_OF_DAY, 0);
|
|
start.set(Calendar.MINUTE, 0);
|
|
start.set(Calendar.SECOND, 0);
|
|
start.set(Calendar.MILLISECOND, 0);
|
|
|
|
Calendar end = Calendar.getInstance();
|
|
end.add(Calendar.DAY_OF_MONTH, -60);
|
|
|
|
String startKeyString = format.format(start.getTime());
|
|
String endKeyString = format.format(end.getTime());
|
|
|
|
JsonArray startKey = generateKey(null, startKeyString);
|
|
JsonArray endKey = generateKey(null, endKeyString);
|
|
|
|
String designDocId = "ServiceUsageRecordAggregated";
|
|
String viewName = "all";
|
|
|
|
|
|
ViewQuery query = ViewQuery.from(designDocId, viewName);
|
|
query.startKey(startKey);
|
|
query.endKey(endKey);
|
|
query.reduce(false);
|
|
query.inclusiveEnd(true);
|
|
|
|
logger.debug("--View Query: startKey:{} - endKey:{} designDocId:{} - viewName:{}",startKey, endKey,designDocId,viewName);
|
|
|
|
|
|
List<Bucket> buckets = new ArrayList<Bucket>();
|
|
|
|
buckets.add(accountingPersistenceQueryCouchBase.connectionMap.get("ServiceUsageRecord"));
|
|
|
|
for(Bucket bucket : buckets){
|
|
|
|
ViewResult viewResult = bucket.query(query);
|
|
|
|
int total = 0;
|
|
int successfullyRemoved = 0;
|
|
|
|
try {
|
|
for (ViewRow row : viewResult) {
|
|
total++;
|
|
|
|
try {
|
|
JsonDocument jsonDocument = row.document();
|
|
|
|
try{
|
|
logger.trace("{}", jsonDocument);
|
|
String id = (String) jsonDocument.content().get("id");
|
|
bucket.remove(id, PersistTo.MASTER, Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS);
|
|
successfullyRemoved++;
|
|
}catch (Throwable e) {
|
|
logger.error("Document {} was not removed", jsonDocument, e);
|
|
}
|
|
}catch (Throwable e) {
|
|
logger.error("Error getting document from row {}. It was not removed", row, e);
|
|
}
|
|
|
|
|
|
}
|
|
} catch(Throwable t){
|
|
logger.info("Total Number of elaborated Document to remove were {}. Successfully removed {}. Difference {}", total, successfullyRemoved, total-successfullyRemoved);
|
|
throw t;
|
|
}
|
|
}
|
|
}
|
|
}
|