package org.gcube.accounting.analytics.persistence.couchbase; /** * */ import java.util.ArrayList; import java.util.Calendar; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Filters; import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TotalFilters; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord; import org.gcube.common.scope.api.ScopeProvider; import org.json.JSONException; import org.json.JSONObject; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author Alessandro Pieve (ISTI - CNR) * */ public class AccountingPersistenceQueryCouchBaseTest { private static Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryCouchBaseTest.class); protected AccountingPersistenceQueryCouchBase accountingPersistenceQueryCouchBase; public class ExtendedInfo extends Info { protected String key; /** * @return the key */ public String getKey() { return key; } /** * @param key the key to set */ public void setKey(String key) { this.key = key; } public ExtendedInfo(String key, Calendar calendar, JSONObject value){ super(calendar, value); this.key = key; } public String toString(){ String info = String .format("Key : %s, %s ", key, super.toString()); return info; } } @Before public void before() throws Exception{ //ScopeProvider.instance.set("/gcube/devNext/NextNext"); ScopeProvider.instance.set("/gcube/devNext"); AccountingPersistenceBackendQueryConfiguration configuration = new AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchBase.class); accountingPersistenceQueryCouchBase = new AccountingPersistenceQueryCouchBase(); accountingPersistenceQueryCouchBase.prepareConnection(configuration); } @After public void after() throws Exception{ //SecurityTokenProvider.instance.reset(); ScopeProvider.instance.reset(); } public void printMap(Map map){ for(Info info : map.values()){ logger.debug("{}", info); } } @Test public void testTopJob() throws Exception { Calendar startTime = Calendar.getInstance(); startTime.set(2015, Calendar.AUGUST, 20); Calendar endTime = Calendar.getInstance(); endTime.set(2016, Calendar.SEPTEMBER, 29,23,59); List filters = new ArrayList(); TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); Class clz = AggregatedJobUsageRecord.class; SortedMap> set = accountingPersistenceQueryCouchBase.getTopValues( clz, temporalConstraint, filters, AggregatedJobUsageRecord.CONSUMER_ID, null); JobUsageRecord record =new JobUsageRecord(); Set result=record.getRequiredFields(); logger.debug("result"+result.toString()); logger.debug("Result final{}", set); } @Test public void testTimeSeries() throws Exception{ Calendar startTime = Calendar.getInstance(); startTime.set(2015, Calendar.AUGUST, 20); Calendar endTime = Calendar.getInstance(); endTime.set(2016, Calendar.SEPTEMBER, 29,23,59); List filters = new ArrayList(); TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); Class clz = AggregatedJobUsageRecord.class; SortedMap set = accountingPersistenceQueryCouchBase.getTimeSeries( clz, temporalConstraint, filters); logger.debug("Result final{}", set); } @Test public void getUsageValue() throws Exception{ Calendar startTime = Calendar.getInstance(); startTime.set(2015, Calendar.MAY, 1); Calendar endTime = Calendar.getInstance(); ScopeProvider.instance.set("/gcube"); Filter filter = new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"); TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); JSONObject filterValue = accountingPersistenceQueryCouchBase.getUsageValue(AggregatedServiceUsageRecord.class, temporalConstraint, filter); logger.info("result:"+filterValue.toString()); } @Test public void getUsageValueQuota() throws Exception{ Calendar startTime = Calendar.getInstance(); startTime.set(2015, Calendar.MAY, 1); Calendar endTime = Calendar.getInstance(); ScopeProvider.instance.set("/gcube"); TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); List filterPackageQuota =new ArrayList(); Filters simpleFilter = new Filters(); List filters=new ArrayList(); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); filters.add(new Filter("serviceClass", "DataAccess")); filters.add(new Filter("serviceName", "CkanConnector")); simpleFilter.setFilters(filters); filterPackageQuota.add(simpleFilter); Filters simpleFilter1 = new Filters(); List filters1=new ArrayList(); filters1.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); filters1.add(new Filter("serviceClass", "VREManagement")); simpleFilter1.setFilters(filters1); filterPackageQuota.add(simpleFilter1); logger.info("filterPackageQuota:"+filterPackageQuota); List filterValue = accountingPersistenceQueryCouchBase.getUsageValueQuota(AggregatedServiceUsageRecord.class, temporalConstraint, filterPackageQuota); logger.info("result:"+filterValue.toString()); } @Test public void getUsageValueQuotaTotal() throws Exception{ Calendar startTime = Calendar.getInstance(); startTime.set(2015, Calendar.MAY, 1); Calendar endTime = Calendar.getInstance(); ScopeProvider.instance.set("/gcube"); TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); /*ask quota for user alessandro pieve*/ List filterPackageQuota =new ArrayList(); Filters simpleFilter = new Filters(); List filters=new ArrayList(); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); filters.add(new Filter("serviceClass", "DataAccess")); filters.add(new Filter("serviceName", "CkanConnector")); simpleFilter.setFilters(filters); filterPackageQuota.add(simpleFilter); Filters simpleFilter1 = new Filters(); List filters1=new ArrayList(); filters1.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); filters1.add(new Filter("serviceClass", "VREManagement")); simpleFilter1.setFilters(filters1); filterPackageQuota.add(simpleFilter1); TotalFilters totalfilter=new TotalFilters(); totalfilter.setTotalFilters(filterPackageQuota); /*ask quota for user lucio lelii*/ List filterPackageQuota1 =new ArrayList(); Filters simpleFilter2 = new Filters(); List filters2=new ArrayList(); filters2.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii")); filters2.add(new Filter("serviceClass", "DataAccess")); filters2.add(new Filter("serviceName", "CkanConnector")); simpleFilter2.setFilters(filters2); filterPackageQuota1.add(simpleFilter2); Filters simpleFilter3 = new Filters(); List filters3=new ArrayList(); filters3.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii")); filters3.add(new Filter("serviceClass", "VREManagement")); simpleFilter3.setFilters(filters3); filterPackageQuota1.add(simpleFilter3); TotalFilters totalfilter1=new TotalFilters(); totalfilter1.setTotalFilters(filterPackageQuota1); List listTotalFilter=new ArrayList(); listTotalFilter.add(totalfilter); listTotalFilter.add(totalfilter1); logger.info("filterPackageQuota:"+listTotalFilter); List filterValue = accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(AggregatedServiceUsageRecord.class, temporalConstraint, listTotalFilter); logger.info("result:"+filterValue.toString()); } @Test public void getQuerableKeyJob() throws Exception{ SortedSet keys; keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedJobUsageRecord.class); for (String key : keys) { if (key != null && !key.isEmpty()) { logger.debug("key:" +key); } } logger.debug("List FilterKeys:" + keys.toString()); } @Test public void testTopService() throws Exception { Calendar startTime = Calendar.getInstance(); startTime.set(2016, Calendar.SEPTEMBER, 27); Calendar endTime = Calendar.getInstance(); endTime.set(2016, Calendar.SEPTEMBER, 28,23,59); List filters = new ArrayList(); TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); Class clz = AggregatedServiceUsageRecord.class; SortedMap> set = accountingPersistenceQueryCouchBase.getTopValues( clz, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, null); logger.debug("Result final{}", set); } @Test public void getQuerableKeyService() throws Exception{ SortedSet keys; keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedServiceUsageRecord.class); for (String key : keys) { if (key != null && !key.isEmpty()) { logger.debug("key:" +key); } } logger.debug("List FilterKeys:" + keys.toString()); } public static SortedMap padMap( SortedMap unpaddedData, TemporalConstraint temporalConstraint) throws Exception { JSONObject jsonObject = getPaddingJSONObject(unpaddedData); SortedSet sequence = temporalConstraint.getCalendarSequence(); for (Calendar progressTime : sequence) { Info info = unpaddedData.get(progressTime); if (info == null) { info = new Info(progressTime, jsonObject); unpaddedData.put(progressTime, info); } } return unpaddedData; } public static JSONObject getPaddingJSONObject( Map unpaddedResults) throws JSONException { JSONObject jsonObject = new JSONObject(); //verify data consistency if (unpaddedResults.size()!=0){ Info auxInfo = new ArrayList(unpaddedResults.values()).get(0); JSONObject auxJsonObject = auxInfo.getValue(); @SuppressWarnings("unchecked") Iterator keys = auxJsonObject.keys(); while (keys.hasNext()) { String key = keys.next(); jsonObject.put(key, 0); } } return jsonObject; } @Test public void testFull() throws Exception { logger.debug("test full "); try{ Calendar startTime = Calendar.getInstance(); //startTime.set(2016, Calendar.AUGUST, 20, 00, 00); startTime.set(2016, Calendar.AUGUST, 29,00,00); Calendar endTime = Calendar.getInstance(); //endTime.set(2016, Calendar.AUGUST, 29, 23, 59); endTime.set(2016, Calendar.AUGUST, 31,23,59); List filters = new ArrayList(); TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); Class clz = AggregatedServiceUsageRecord.class; SortedMap> set = accountingPersistenceQueryCouchBase.getTopValues( clz, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, null); /**pad*/ int limit=0; boolean pad=true; int count = set.size() > limit ? limit : set.size(); NumberedFilter firstRemovalKey = null; logger.debug("set completo"+set.toString()); for(NumberedFilter nf : set.keySet()){ if(--count>=0 || limit<=0){ if(pad){ padMap(set.get(nf), temporalConstraint); } }else{ if(firstRemovalKey==null){ firstRemovalKey = nf; }else{ break; } } } if(firstRemovalKey!=null){ logger.debug("First removal key set:"+set.subMap(set.firstKey(), firstRemovalKey)); } logger.debug("set: "+set); }catch(Exception e){ e.printStackTrace(); } } @Test public void testContextService() throws Exception { Calendar startTime = Calendar.getInstance(); startTime.set(2016, Calendar.SEPTEMBER, 1); Calendar endTime = Calendar.getInstance(); endTime.set(2016, Calendar.OCTOBER, 20,23,59); List filters = new ArrayList(); //filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); Class clz = AggregatedServiceUsageRecord.class; List context=new ArrayList(); context.add("/gcube"); context.add("/gcube/devNext"); context.add("/gcube/devNext/nextNext"); context.add("/gcube/devsec"); context.add("/gcube/devsec/devVRE"); context.add("/gcube/devsec/preVRE"); context.add("/gcube/preprod/preVRE"); context.add("/gcube/preprod"); SortedMap> setContext = accountingPersistenceQueryCouchBase.getContextTimeSeries( clz, temporalConstraint, filters, context); logger.debug("Result final{}", setContext); } }