package org.gcube.accounting.analytics.persistence.couchbase; import java.text.DateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.concurrent.TimeUnit; import org.gcube.accounting.aggregator.aggregation.AggregationType; import org.gcube.accounting.aggregator.utility.Utility; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.testutility.ContextTest; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.couchbase.client.java.Bucket; import com.couchbase.client.java.PersistTo; import com.couchbase.client.java.document.JsonDocument; import com.couchbase.client.java.document.json.JsonArray; import com.couchbase.client.java.view.ViewQuery; import com.couchbase.client.java.view.ViewResult; import com.couchbase.client.java.view.ViewRow; public class RemoveOldRecords extends ContextTest { private static final Logger logger = LoggerFactory.getLogger(RemoveOldRecords.class); protected static JsonArray generateKey(String scope,String key){ JsonArray generateKey = JsonArray.create(); if (scope!=null){ generateKey.add(scope); } for (String value: key.split("/")){ if (!value.toString().isEmpty()) generateKey.add(Integer.parseInt(value)); } return generateKey; } @Test public void removeOldRecords() throws Exception{ AccountingPersistenceBackendQueryConfiguration configuration = new AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchBase.class); AccountingPersistenceQueryCouchBase accountingPersistenceQueryCouchBase = new AccountingPersistenceQueryCouchBase(); accountingPersistenceQueryCouchBase.prepareConnection(configuration); DateFormat format = Utility.getUTCDateFormat(AggregationType.DAILY.getDateFormatPattern()); Calendar start = Calendar.getInstance(); start.set(Calendar.YEAR, 1970); start.set(Calendar.MONTH, Calendar.JANUARY); start.set(Calendar.DAY_OF_MONTH, 1); start.set(Calendar.HOUR_OF_DAY, 0); start.set(Calendar.MINUTE, 0); start.set(Calendar.SECOND, 0); start.set(Calendar.MILLISECOND, 0); Calendar end = Calendar.getInstance(); end.add(Calendar.DAY_OF_MONTH, -60); String startKeyString = format.format(start.getTime()); String endKeyString = format.format(end.getTime()); JsonArray startKey = generateKey(null, startKeyString); JsonArray endKey = generateKey(null, endKeyString); String designDocId = "ServiceUsageRecordAggregated"; String viewName = "all"; ViewQuery query = ViewQuery.from(designDocId, viewName); query.startKey(startKey); query.endKey(endKey); query.reduce(false); query.inclusiveEnd(true); logger.debug("--View Query: startKey:{} - endKey:{} designDocId:{} - viewName:{}",startKey, endKey,designDocId,viewName); List buckets = new ArrayList(); buckets.add(accountingPersistenceQueryCouchBase.connectionMap.get("ServiceUsageRecord")); for(Bucket bucket : buckets){ ViewResult viewResult = bucket.query(query); int total = 0; int successfullyRemoved = 0; try { for (ViewRow row : viewResult) { total++; try { JsonDocument jsonDocument = row.document(); try{ logger.trace("{}", jsonDocument); String id = (String) jsonDocument.content().get("id"); bucket.remove(id, PersistTo.MASTER, Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); successfullyRemoved++; }catch (Throwable e) { logger.error("Document {} was not removed", jsonDocument, e); } }catch (Throwable e) { logger.error("Error getting document from row {}. It was not removed", row, e); } } } catch(Throwable t){ logger.info("Total Number of elaborated Document to remove were {}. Successfully removed {}. Difference {}", total, successfullyRemoved, total-successfullyRemoved); throw t; } } } }