diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/Constant.java b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/Constant.java new file mode 100644 index 0000000..8079215 --- /dev/null +++ b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/Constant.java @@ -0,0 +1,17 @@ +package org.gcube.accounting.analytics.persistence.couchbase; + + +/** + * @author Alessandro Pieve (ISTI - CNR) + * + */ + +public class Constant { + + public static final Integer CONNECTION_TIMEOUT=15; + public static final Integer NUM_RETRY=6; + public static final Integer CONNECTION_TIMEOUT_BUCKET=15; + public static final Integer VIEW_TIMEOUT_BUCKET=120; + public static final Integer MAX_REQUEST_LIFE_TIME=120; + +} diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/RemoveOldRecords.java b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/RemoveOldRecords.java new file mode 100644 index 0000000..d4afe50 --- /dev/null +++ b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/RemoveOldRecords.java @@ -0,0 +1,111 @@ +package org.gcube.accounting.analytics.persistence.couchbase; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.concurrent.TimeUnit; + +import org.gcube.accounting.aggregator.madeaggregation.AggregationType; +import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; +import org.gcube.testutility.ScopedTest; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.PersistTo; +import com.couchbase.client.java.document.JsonDocument; +import com.couchbase.client.java.document.json.JsonArray; +import com.couchbase.client.java.view.ViewQuery; +import com.couchbase.client.java.view.ViewResult; +import com.couchbase.client.java.view.ViewRow; + +public class RemoveOldRecords extends ScopedTest { + + private static final Logger logger = LoggerFactory.getLogger(RemoveOldRecords.class); + + protected static JsonArray generateKey(String scope,String key){ + JsonArray generateKey = JsonArray.create(); + if (scope!=null){ + generateKey.add(scope); + } + for (String value: key.split(",")){ + if (!value.toString().isEmpty()) + generateKey.add(Integer.parseInt(value)); + } + return generateKey; + + } + + @Test + public void removeOldRecords() throws Exception{ + AccountingPersistenceBackendQueryConfiguration configuration = new + AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchBase.class); + AccountingPersistenceQueryCouchBase accountingPersistenceQueryCouchBase = new AccountingPersistenceQueryCouchBase(); + accountingPersistenceQueryCouchBase.prepareConnection(configuration); + + SimpleDateFormat format = new SimpleDateFormat(AggregationType.HOURLY.getDateformat()); + + Calendar start = Calendar.getInstance(); + start.set(1970, 1, 1); + Calendar end = Calendar.getInstance(); + end.add(Calendar.DAY_OF_MONTH, -60); + + String startKeyString = format.format(start.getTime()); + String endKeyString = format.format(end.getTime()); + + JsonArray startKey = generateKey(null, startKeyString); + JsonArray endKey = generateKey(null, endKeyString); + + String designDocId = "noContext"; + String viewName = "all"; + + + ViewQuery query = ViewQuery.from(designDocId, viewName); + query.startKey(startKey); + query.endKey(endKey); + query.reduce(false); + query.inclusiveEnd(true); + + logger.debug("--View Query: startKey:{} - endKey:{} designDocId:{} - viewName:{}",startKey, endKey,designDocId,viewName); + + + Bucket[] buckets = new Bucket[] { + accountingPersistenceQueryCouchBase.bucketService, + accountingPersistenceQueryCouchBase.bucketStorage + }; + + for(Bucket bucket : buckets){ + + ViewResult viewResult = bucket.query(query); + + int total = 0; + int successfullyRemoved = 0; + + try { + for (ViewRow row : viewResult) { + total++; + + try { + JsonDocument jsonDocument = row.document(); + + try{ + logger.trace("{}", jsonDocument); + String id = (String) jsonDocument.content().get("id"); + bucket.remove(id, PersistTo.MASTER, Constant.CONNECTION_TIMEOUT_BUCKET, TimeUnit.SECONDS); + successfullyRemoved++; + }catch (Throwable e) { + logger.error("Document {} was not removed", jsonDocument, e); + } + }catch (Throwable e) { + logger.error("Error getting document from row {}. It was not removed", row, e); + } + + + } + } catch(Throwable t){ + logger.info("Total Number of elaborated Document to remove were {}. Successfully removed {}. Difference {}", total, successfullyRemoved, total-successfullyRemoved); + throw t; + } + } + } +}