From 3fcc59c6aee4620febe3e671e05845e09cbaf26c Mon Sep 17 00:00:00 2001 From: Alessandro Pieve Date: Tue, 15 Nov 2016 10:41:07 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics-persistence-couchbase@134163 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../AccountingPersistenceQueryCouchBase.java | 400 ++++++------------ ...countingPersistenceQueryCouchBaseTest.java | 172 ++++---- 2 files changed, 220 insertions(+), 352 deletions(-) diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java b/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java index ad1a8b9..a7880e3 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java @@ -24,13 +24,14 @@ import java.util.TreeMap; import java.util.TreeSet; import org.gcube.accounting.analytics.Filter; -import org.gcube.accounting.analytics.Filters; +import org.gcube.accounting.analytics.FiltersValue; import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.TemporalConstraint.CalendarEnum; -import org.gcube.accounting.analytics.TotalFilters; +import org.gcube.accounting.analytics.UsageServiceValue; +import org.gcube.accounting.analytics.UsageValue; import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException; import org.gcube.accounting.analytics.exception.ValueException; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; @@ -38,6 +39,7 @@ import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQu import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.datamodel.BasicUsageRecord; import org.gcube.accounting.datamodel.UsageRecord; +import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.documentstore.records.AggregatedRecord; @@ -75,10 +77,7 @@ AccountingPersistenceBackendQuery { private static final Logger logger = LoggerFactory .getLogger(AccountingPersistenceQueryCouchBase.class); - public static final String URL_PROPERTY_KEY = AccountingPersistenceConfiguration.URL_PROPERTY_KEY; - // public static final String USERNAME_PROPERTY_KEY = - // AccountingPersistenceConfiguration.USERNAME_PROPERTY_KEY; public static final String PASSWORD_PROPERTY_KEY = AccountingPersistenceConfiguration.PASSWORD_PROPERTY_KEY; public static final String BUCKET_STORAGE_NAME_PROPERTY_KEY="AggregatedStorageUsageRecord"; @@ -127,8 +126,6 @@ AccountingPersistenceBackendQuery { String password = configuration.getProperty(PASSWORD_PROPERTY_KEY); cluster = CouchbaseCluster.create(ENV, url); - logger.trace("env"+ENV.toString()); - bucketNameStorage = configuration.getProperty(BUCKET_STORAGE_NAME_PROPERTY_KEY); bucketNameService = configuration.getProperty(BUCKET_SERVICE_NAME_PROPERTY_KEY); bucketNameJob = configuration.getProperty(BUCKET_JOB_NAME_PROPERTY_KEY); @@ -152,8 +149,7 @@ AccountingPersistenceBackendQuery { bucketTask= cluster.openBucket(bucketNameTask, password); connectionMap.put(BUCKET_TASK_NAME_PROPERTY_KEY, bucketTask); - logger.trace("Open cluster Service Bucket Url:"+url+" Pwd:"+configuration.getProperty(PASSWORD_PROPERTY_KEY)+ - " BucketName "+configuration.getProperty(BUCKET_SERVICE_NAME_PROPERTY_KEY)); + logger.trace("Open cluster Service Bucket Url:"+url+" BucketName:"+configuration.getProperty(BUCKET_SERVICE_NAME_PROPERTY_KEY)); } /** @@ -190,7 +186,6 @@ AccountingPersistenceBackendQuery { TemporalConstraint temporalConstraint, List filters) throws Exception { String currentScope = ScopeProvider.instance.get(); - String recordType = clz.newInstance().getRecordType(); Expression expression = x(BasicUsageRecord.SCOPE).eq(s(currentScope)); @@ -221,9 +216,7 @@ AccountingPersistenceBackendQuery { .where(expression); Map map = new HashMap(); - //logger.info("result"+result.toString()); N1qlQueryResult result = connectionMap.get(clz.getSimpleName()).query(groupByPath); - if (!result.finalSuccess()) { logger.debug("{} failed : {}", N1qlQueryResult.class.getSimpleName(), result.errors()); @@ -278,17 +271,13 @@ AccountingPersistenceBackendQuery { count++; startFound = true; } catch (Exception e) { - /* - logger.trace("The provide value is not an int. {}", array - .get(i).toString()); - */ + //logger.trace("The provide value is not an int. {}", array.get(i).toString()); if (startFound) { break; } } } - for (int j = count; j < calendarValues.length; j++) { if (calendarValues[j].getCalendarValue() == Calendar.DAY_OF_MONTH) { calendar.set(calendarValues[j].getCalendarValue(), 1); @@ -296,9 +285,7 @@ AccountingPersistenceBackendQuery { calendar.set(calendarValues[j].getCalendarValue(), 0); } } - return calendar; - } protected JsonArray getRangeKey(long time, AggregationMode aggregationMode, @@ -308,26 +295,19 @@ AccountingPersistenceBackendQuery { Calendar calendar = Calendar.getInstance(); calendar.setTimeInMillis(time); CalendarEnum[] values = CalendarEnum.values(); - if (endKey) { - calendar.add(values[aggregationMode.ordinal()].getCalendarValue(), - 1); + calendar.add(values[aggregationMode.ordinal()].getCalendarValue(),1); } - for (int i = 0; i <= aggregationMode.ordinal(); i++) { int value = calendar.get(values[i].getCalendarValue()); - if (values[i].getCalendarValue() == Calendar.MONTH) { value = value + 1; } - array.add(value); } - if (wildCard) { array.add("{}"); } - return array; } @@ -339,33 +319,31 @@ AccountingPersistenceBackendQuery { * Used in the name of map reduce to separate keys used as filter */ protected static final String KEYS_SEPARATOR = "__"; - protected String getDesignDocId( Class> recordClass) throws InstantiationException, IllegalAccessException { - - String getDesigndocid=String.format("%s%s", MAP_REDUCE__DESIGN, recordClass - .newInstance().getRecordType()); - //logger.debug("use a designDocID:{}",getDesigndocid); - - return String.format("%s%s", MAP_REDUCE__DESIGN, recordClass .newInstance().getRecordType()); } - + /** + * + * @param recordClass + * @param keys + * @return + * @throws InstantiationException + * @throws IllegalAccessException + */ protected String getDesignDocIdSpecific( Class> recordClass, Collection keys) throws InstantiationException, IllegalAccessException { - String getDesigndocId=String.format("%s%s", MAP_REDUCE__DESIGN, recordClass - .newInstance().getRecordType()); String specific="all"; if (!keys.isEmpty()){ specific = keys.iterator().next(); } - String getDesigndocIdSpecific=getDesigndocId+SEPARATOR_DESIGN_DOC+specific; - logger.debug("use a designDocIDSpecific:{}",getDesigndocIdSpecific); + String getDesigndocIdSpecific=specific; + logger.trace("Use a designDocIDSpecific:{}",getDesigndocIdSpecific); return getDesigndocIdSpecific; } @@ -464,10 +442,9 @@ AccountingPersistenceBackendQuery { groupLevel += keys.size(); } - String designDocId = getDesignDocId(clz); - //ADD A SPECIFY DESIGN DOC FAMILY - String designDocIdNew = getDesignDocIdSpecific(clz,keys); - logger.trace("designDocIdNew :{}",designDocIdNew); + //String designDocId = getDesignDocId(clz); + String designDocId = getDesignDocIdSpecific(clz,keys); + logger.trace("designDocIdNew :{}",designDocId); for (Object temporal: temporalStartKey.toList()){ if (!temporal.toString().isEmpty()) startKey.add(temporal); @@ -509,7 +486,6 @@ AccountingPersistenceBackendQuery { JsonArray array = (JsonArray) row.key(); Calendar calendar = getCalendarFromArray(array); - JsonObject value = (JsonObject) row.value(); JSONObject obj = new JSONObject(value.toString()); Info info = new Info(calendar, obj); @@ -518,8 +494,6 @@ AccountingPersistenceBackendQuery { } if (infos.isEmpty()){ - - //exec a map reduce for found name key query = ViewQuery.from(designDocId, viewName); query.groupLevel(groupLevel); query.descending(false); @@ -556,10 +530,10 @@ AccountingPersistenceBackendQuery { return infos; } - - - - + + + + @Override public SortedMap getTimeSeries( Class> clz, @@ -570,7 +544,7 @@ AccountingPersistenceBackendQuery { return map; } - + @Override public SortedMap> getTopValues( Class> clz, @@ -610,18 +584,16 @@ AccountingPersistenceBackendQuery { @Override public SortedMap> getContextTimeSeries( Class> clz, - TemporalConstraint temporalConstraint, List filters,List contexts) - throws Exception { - logger.trace("getContextTimeSeries"); + TemporalConstraint temporalConstraint, List filters,List contexts) + throws Exception { + logger.trace("getContextTimeSeries for contexts:{}",contexts.toString()); SortedSet listContexts = new TreeSet(); for (String context:contexts){ Filter contextLabel= new Filter("context",context); listContexts.add(contextLabel); } SortedMap> ret = new TreeMap<>(); - - for(Filter nf : listContexts){ - //filters.add(nf); + for(Filter nf : listContexts){ logger.debug("detail time series :{}",nf.toString()); SortedMap map = mapReduceQuery(clz, temporalConstraint, filters,nf.getValue()); @@ -629,7 +601,7 @@ AccountingPersistenceBackendQuery { filters.remove(nf); } return ret; - + } protected String getQualifiedProperty(String property){ @@ -651,7 +623,7 @@ AccountingPersistenceBackendQuery { String key, String orderingProperty) throws Exception { String currentScope = ScopeProvider.instance.get(); - String recordType = clz.newInstance().getRecordType(); + //String recordType = clz.newInstance().getRecordType(); if(orderingProperty==null){ orderingProperty = AccountingPersistenceQuery. @@ -659,19 +631,12 @@ AccountingPersistenceBackendQuery { } Collection selectExpressions = new ArrayList<>(); - //add select expression - /* - selectExpressions.add(x("SUM(" + getSpecializedProperty(clz,orderingProperty) + ")"). - as(orderingProperty)); - */ + //selectExpressions.add(x("SUM(" + getSpecializedProperty(clz,orderingProperty) + ")").as(orderingProperty)); //add select expression and check if exist - selectExpressions.add(x("SUM(CASE WHEN " + getSpecializedProperty(clz,orderingProperty) + " IS NOT NULL THEN "+getSpecializedProperty(clz,orderingProperty)+" ELSE 1 END )"). as(orderingProperty)); - - //selectExpressions.add(x(getSpecializedProperty(clz,key)).as(key)); selectExpressions.add(x("(CASE WHEN " + getSpecializedProperty(clz,key) + " IS NOT NULL THEN "+getSpecializedProperty(clz,key)+" ELSE 'UNKNOWN' END )").as(key)); @@ -686,7 +651,6 @@ AccountingPersistenceBackendQuery { x(getSpecializedProperty(clz,AggregatedRecord.START_TIME)).gt(startTime) ); //long endTime = temporalConstraint.getAlignedEndTime().getTimeInMillis(); - //if (startTime==endTime) long endTime = temporalConstraint.getEndTime(); whereExpression = whereExpression.and( @@ -739,7 +703,7 @@ AccountingPersistenceBackendQuery { OffsetPath path = select(selectExpressionArray).from(connectionMap.get(clz.getSimpleName()).name()) .where(whereExpression).groupBy(key).orderBy(sort); - logger.debug("Query for top"+path.toString()); + logger.debug("Query for top:"+path.toString()); Comparator comparator = new Comparator() { @Override public int compare(NumberedFilter o1, NumberedFilter o2) { @@ -803,15 +767,12 @@ AccountingPersistenceBackendQuery { int groupLevel = scopeDateGroupLevel; //NO ADD A SPECIFIY DESIGN DOC ID FAMILY String designDocId = getDesignDocId(clz)+"Value"; - + String viewName = key; - logger.trace("designDocId:{} view:{} ",designDocId,key); - logger.trace("startKey:{}",startKey); - logger.trace("groupLevel"+groupLevel); + logger.trace("designDocId:{} view:{} startKey:{} groupLevel:{}",designDocId,key,startKey,groupLevel); ViewQuery query = ViewQuery.from(designDocId, viewName); query.inclusiveEnd(); - query.groupLevel(groupLevel); query.startKey(startKey); query.descending(false); @@ -859,7 +820,6 @@ AccountingPersistenceBackendQuery { Class> clz, TemporalConstraint temporalConstraint, Filter applicant) throws Exception { - String currentScope = ScopeProvider.instance.get(); JsonArray startKey = JsonArray.create(); @@ -868,7 +828,6 @@ AccountingPersistenceBackendQuery { JsonArray endKey = JsonArray.create(); endKey.add(currentScope); - AggregationMode aggregationMode = temporalConstraint .getAggregationMode(); @@ -882,13 +841,13 @@ AccountingPersistenceBackendQuery { startKey.add(applicant.getValue()); - for (Object temporal: temporalStartKey.toList()){ if (!temporal.toString().isEmpty()) startKey.add(temporal); } endKey.add(applicant.getValue()); + int count =1; for (Object temporal: temporalEndKey.toList()){ if (!temporal.toString().isEmpty()){ @@ -906,12 +865,16 @@ AccountingPersistenceBackendQuery { int scopeDateGroupLevel = aggregationMode.ordinal() + 1 + 1 +1; int groupLevel = scopeDateGroupLevel; - String designDocId = getDesignDocId(clz); + + Collection keys = new TreeSet<>(); + keys.add(applicant.getKey()); + //String designDocId = getDesignDocId(clz); //ADD A SPECIFIY DESIGN DOC ID FAMILY + String designDocId = getDesignDocIdSpecific(clz,keys); + String viewName = applicant.getKey(); ViewQuery query = ViewQuery.from(designDocId, viewName); - query.inclusiveEnd(); query.groupLevel(groupLevel); query.startKey(startKey); @@ -958,173 +921,55 @@ AccountingPersistenceBackendQuery { return result; } - @Override - public List getUsageValueQuota( - Class> clz, - TemporalConstraint temporalConstraint, - List filterPackageQuota) throws Exception { - - - String currentScope = ScopeProvider.instance.get(); - AggregationMode aggregationMode = temporalConstraint - .getAggregationMode(); - - JsonArray temporalStartKey = getRangeKey( - temporalConstraint.getStartTime(), - aggregationMode, false, false); - - JsonArray temporalEndKey = getRangeKey( - temporalConstraint.getEndTime(), - aggregationMode, false, false); - - String designDocId = getDesignDocId(clz); - //ADD A SPECIFIY DESIGN DOC ID FAMILY - for (Filters singleFilter:filterPackageQuota){ - String viewNameTmp=null; - - JsonArray startKeyTmp=JsonArray.create(); - startKeyTmp.add(currentScope); - - JsonArray endKeyTmp=JsonArray.create(); - endKeyTmp.add(currentScope); - - int groupLevelTmp= 2; - - for (Filter filter:singleFilter.getFilters()){ - if (groupLevelTmp==2) - viewNameTmp=filter.getKey(); - else - viewNameTmp=viewNameTmp+"__"+filter.getKey(); - startKeyTmp.add(filter.getValue()); - endKeyTmp.add(filter.getValue()); - groupLevelTmp++; - - } - - for (Object temporal: temporalStartKey.toList()){ - if (!temporal.toString().isEmpty()) - startKeyTmp.add(temporal); - } - int count =1; - for (Object temporal: temporalEndKey.toList()){ - if (!temporal.toString().isEmpty()){ - //couchbase exclude last value - if (count==temporalEndKey.size()) - temporal=(int)temporal+1; - endKeyTmp.add(temporal); - } - count++; - } - - ViewQuery query = ViewQuery.from(designDocId, viewNameTmp); - query.inclusiveEnd(); - query.groupLevel(groupLevelTmp); - query.startKey(startKeyTmp); - query.endKey(endKeyTmp); - query.descending(false); - - logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, " - + "Group Level : {}, Start Key : {}, End Key : {}," - + "temporalStartKey :{}, temporalEndKey :{}", - clz.getSimpleName(),designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp,temporalStartKey.toString(), temporalEndKey.toString()); - - ViewResult viewResult; - try { - //execute query in a specify bucket - viewResult = connectionMap.get(clz.getSimpleName()).query(query); - - } catch (Exception e) { - logger.error(e.getLocalizedMessage()); - throw e; - } - - Map map = new HashMap(); - - for (ViewRow row : viewResult) { - - JsonObject jsnobject = (JsonObject) row.value(); - JSONObject objJson = new JSONObject(jsnobject.toString()); - - Iterator iterateJosn = objJson.keys(); - while( iterateJosn.hasNext() ) { - String key = (String)iterateJosn.next(); - Float valuetmp=Float.parseFloat(objJson.get(key).toString()); - - if (key.equals("operationCount") || key.equals("dataVolume")){ - if (map.containsKey(key)) { - map.put(key, valuetmp + map.get(key)); - - // TODO verify a better method - if (designDocId.equals("StorageUsageRecord")){ - if (key.equals("dataVolume")){ - singleFilter.setOrderingProperty(key); - singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue()); - } - } - else{ - singleFilter.setOrderingProperty(key); - singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue()); - } - - } - else{ - map.put(key, valuetmp); - - // TODO verify a better method - if (designDocId.equals("StorageUsageRecord")){ - if (key.equals("dataVolume")){ - singleFilter.setOrderingProperty(key); - singleFilter.setD(valuetmp.doubleValue()); - } - } - else{ - singleFilter.setOrderingProperty(key); - singleFilter.setD(valuetmp.doubleValue()); - } - - } - - } - } - } - } - return filterPackageQuota; - } - - - /** * */ @Override - public List getUsageValueQuotaTotal( - Class> clz, - TemporalConstraint temporalConstraint, List listUsage) - throws Exception { + public List getUsageValueQuotaTotal( + List listUsage) + throws Exception { + //String currentScope = ScopeProvider.instance.get(); + String keyOrderingProperty = null; + for (UsageValue totalFilters:listUsage){ - String currentScope = ScopeProvider.instance.get(); - AggregationMode aggregationMode = temporalConstraint - .getAggregationMode(); + String currentScope = totalFilters.getContext(); - JsonArray temporalStartKey = getRangeKey( - temporalConstraint.getStartTime(), - aggregationMode, false, false); + Collection keys= new TreeSet<>(); + keys.add("consumerId"); + String designDocId = getDesignDocIdSpecific(totalFilters.getClz(), keys); - JsonArray temporalEndKey = getRangeKey( - temporalConstraint.getEndTime(), - aggregationMode, false, false); + JsonArray temporalStartKey=null; + JsonArray temporalEndKey=null; + TemporalConstraint temporalConstraint=totalFilters.getTemporalConstraint(); + if (temporalConstraint==(null)){ + logger.trace("Not found temporalConstraint"); + Calendar startTime = Calendar.getInstance(); + startTime.set(1970, Calendar.JANUARY, 1); + Calendar endTime = Calendar.getInstance(); + temporalConstraint = + new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + } - String designDocId = getDesignDocId(clz); - //ADD A SPECIFIC DESIGN DOC ID FAMILY - - for (TotalFilters totalfilters:listUsage){ + AggregationMode aggregationMode = temporalConstraint.getAggregationMode(); + temporalStartKey = getRangeKey( + temporalConstraint.getStartTime(), + aggregationMode, false, false); + temporalEndKey = getRangeKey( + temporalConstraint.getEndTime(), + aggregationMode, false, false); Double totalQuota=0.00; - - for (Filters singleFilter:totalfilters.getTotalFilters()){ - String viewNameTmp=null; + int i = 0; + int countFilters=0; + if (totalFilters.getClass().getSimpleName().equals(UsageServiceValue.class.getSimpleName())){ + UsageServiceValue totalFiltersService = (UsageServiceValue)totalFilters; + countFilters =totalFiltersService.getFiltersValue().size(); + } + do { + String viewNameTmp=null; JsonArray startKeyTmp=JsonArray.create(); startKeyTmp.add(currentScope); @@ -1132,18 +977,22 @@ AccountingPersistenceBackendQuery { endKeyTmp.add(currentScope); int groupLevelTmp= 2; - - for (Filter filter:singleFilter.getFilters()){ - if (groupLevelTmp==2) - viewNameTmp=filter.getKey(); - else + //FiltersValue singleFilter=null; + viewNameTmp=AggregatedServiceUsageRecord.CONSUMER_ID; + startKeyTmp.add(totalFilters.getIdentifier()); + endKeyTmp.add(totalFilters.getIdentifier()); + FiltersValue singleFilter=null; + if (totalFilters.getClass().getSimpleName().equals(UsageServiceValue.class.getSimpleName())){ + UsageServiceValue totalFiltersService = (UsageServiceValue)totalFilters; + singleFilter=totalFiltersService.getFiltersValue().get(i); + for (Filter filter:singleFilter.getFiltersValue()){ viewNameTmp=viewNameTmp+"__"+filter.getKey(); - startKeyTmp.add(filter.getValue()); - endKeyTmp.add(filter.getValue()); - groupLevelTmp++; - + startKeyTmp.add(filter.getValue()); + endKeyTmp.add(filter.getValue()); + groupLevelTmp++; + } } - + //not defined temporal constraint for (Object temporal: temporalStartKey.toList()){ if (!temporal.toString().isEmpty()) startKeyTmp.add(temporal); @@ -1151,13 +1000,17 @@ AccountingPersistenceBackendQuery { int count =1; for (Object temporal: temporalEndKey.toList()){ if (!temporal.toString().isEmpty()){ - //couchbase exclude last value + //couchbase excludes last value if (count==temporalEndKey.size()) temporal=(int)temporal+1; endKeyTmp.add(temporal); } count++; } + logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, " + + "Group Level : {}, Start Key : {}, End Key : {}," + + "temporalStartKey :{}, temporalEndKey :{}", + totalFilters.getClz().getSimpleName(),designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp,temporalStartKey.toString(), temporalEndKey.toString()); ViewQuery query = ViewQuery.from(designDocId, viewNameTmp); query.inclusiveEnd(); @@ -1166,16 +1019,9 @@ AccountingPersistenceBackendQuery { query.endKey(endKeyTmp); query.descending(false); - logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, " - + "Group Level : {}, Start Key : {}, End Key : {}," - + "temporalStartKey :{}, temporalEndKey :{}", - clz.getSimpleName(),designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp,temporalStartKey.toString(), temporalEndKey.toString()); - ViewResult viewResult; try { - //execute query in a specify bucket - viewResult = connectionMap.get(clz.getSimpleName()).query(query); - + viewResult = connectionMap.get(totalFilters.getClz().getSimpleName()).query(query); } catch (Exception e) { logger.error(e.getLocalizedMessage()); throw e; @@ -1191,23 +1037,31 @@ AccountingPersistenceBackendQuery { Iterator iterateJosn = objJson.keys(); while( iterateJosn.hasNext() ) { String key = (String)iterateJosn.next(); + Float valuetmp=Float.parseFloat(objJson.get(key).toString()); if (key.equals("operationCount") || key.equals("dataVolume")){ + if (map.containsKey(key)) { map.put(key, valuetmp + map.get(key)); // TODO verify a better method if (designDocId.equals("StorageUsageRecord")){ if (key.equals("dataVolume")){ - singleFilter.setOrderingProperty(key); - singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue()); + if (totalFilters.getClass().getSimpleName().equals(UsageServiceValue.class.getSimpleName())){ + singleFilter.setOrderingProperty(key); + singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue()); + } + keyOrderingProperty=key; totalQuota+=singleFilter.getD()+valuetmp.doubleValue(); } } else{ - singleFilter.setOrderingProperty(key); - singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue()); + if (totalFilters.getClass().getSimpleName().equals(UsageServiceValue.class.getSimpleName())){ + singleFilter.setOrderingProperty(key); + singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue()); + } + keyOrderingProperty=key; totalQuota+=singleFilter.getD()+valuetmp.doubleValue(); } @@ -1218,36 +1072,32 @@ AccountingPersistenceBackendQuery { // TODO verify a better method if (designDocId.equals("StorageUsageRecord")){ if (key.equals("dataVolume")){ - singleFilter.setOrderingProperty(key); - singleFilter.setD(valuetmp.doubleValue()); + keyOrderingProperty=key; + if (totalFilters.getClass().getSimpleName().equals(UsageServiceValue.class.getSimpleName())){ + singleFilter.setOrderingProperty(key); + singleFilter.setD(valuetmp.doubleValue()); + } totalQuota+=valuetmp.doubleValue(); } } else{ - singleFilter.setOrderingProperty(key); - singleFilter.setD(valuetmp.doubleValue()); + keyOrderingProperty=key; + if (totalFilters.getClass().getSimpleName().equals(UsageServiceValue.class.getSimpleName())){ + singleFilter.setOrderingProperty(key); + singleFilter.setD(valuetmp.doubleValue()); + } totalQuota+=valuetmp.doubleValue(); } - } - } } } - } - totalfilters.setD(totalQuota); - + i++; + } while (i filterPackageQuota =new ArrayList(); - - - Filters simpleFilter = new Filters(); - List filters=new ArrayList(); - filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); - filters.add(new Filter("serviceClass", "DataAccess")); - filters.add(new Filter("serviceName", "CkanConnector")); - simpleFilter.setFilters(filters); - - filterPackageQuota.add(simpleFilter); - - - Filters simpleFilter1 = new Filters(); - List filters1=new ArrayList(); - filters1.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); - filters1.add(new Filter("serviceClass", "VREManagement")); - simpleFilter1.setFilters(filters1); - filterPackageQuota.add(simpleFilter1); - - logger.info("filterPackageQuota:"+filterPackageQuota); - List filterValue = - accountingPersistenceQueryCouchBase.getUsageValueQuota(AggregatedServiceUsageRecord.class, temporalConstraint, filterPackageQuota); - logger.info("result:"+filterValue.toString()); - } - - + + /** + * utilizzato per effettuare una singola chiamata ad accounting analytics in modo che ritorni una lista di quote riempite + * + * OUTPUT: + * result:[ + * + * [ + * UsageServiceValue [ + * clz=class org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord, + * temporalConstraint=StartTime : 2015-05-01 10:25:49:238 UTC (1430475949238 millis), EndTime : 2016-11-11 10:25:49:238 UTC (1478859949238 millis), + * Aggregated DAILY, + * filtersValue=[ + * FiltersValue [ + * filters=[{ "serviceClass" : "DataAccess" }, { "serviceName" : "CkanConnector" }], d=1.0, orderingProperty=operationCount], + * FiltersValue [ + * filters=[{ "serviceClass" : "VREManagement" }], d=1.0, orderingProperty=operationCount] + * ], + * identifier=lucio.lelii, + * d=2.0, + * orderingProperty=operationCount + * ], + * + * UsageStorageValue [ + * clz=class org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord, + * temporalConstraint=StartTime : 2015-05-01 10:25:49:238 UTC (1430475949238 millis), EndTime : 2016-11-11 10:25:49:238 UTC (1478859949238 millis), + * Aggregated DAILY, + * identifier=alessandro.pieve, + * d=714216.0, + * orderingProperty=dataVolume + * ] + * ] + */ @Test public void getUsageValueQuotaTotal() throws Exception{ Calendar startTime = Calendar.getInstance(); @@ -232,64 +229,97 @@ public class AccountingPersistenceQueryCouchBaseTest { ScopeProvider.instance.set("/gcube"); - + String context="/gcube"; TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); - /*ask quota for user alessandro pieve*/ - List filterPackageQuota =new ArrayList(); + /*ask quota for user alessandro pieve + * + * SERVICE + * */ + List filterPackageQuota =new ArrayList(); - Filters simpleFilter = new Filters(); + FiltersValue simpleFilter = new FiltersValue(); List filters=new ArrayList(); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); filters.add(new Filter("serviceClass", "DataAccess")); filters.add(new Filter("serviceName", "CkanConnector")); - simpleFilter.setFilters(filters); + simpleFilter.setFiltersValue(filters); filterPackageQuota.add(simpleFilter); - Filters simpleFilter1 = new Filters(); + FiltersValue simpleFilter1 = new FiltersValue(); List filters1=new ArrayList(); filters1.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); filters1.add(new Filter("serviceClass", "VREManagement")); - simpleFilter1.setFilters(filters1); + simpleFilter1.setFiltersValue(filters1); filterPackageQuota.add(simpleFilter1); - TotalFilters totalfilter=new TotalFilters(); - totalfilter.setTotalFilters(filterPackageQuota); + UsageValue totalfilter=new UsageServiceValue(context,"lucio.lelii",AggregatedServiceUsageRecord.class,temporalConstraint,filterPackageQuota); - /*ask quota for user lucio lelii*/ - List filterPackageQuota1 =new ArrayList(); - Filters simpleFilter2 = new Filters(); + /*ask quota for user lucio lelii + * SERVICE + * */ + List filterPackageQuota1 =new ArrayList(); + + FiltersValue simpleFilter2 = new FiltersValue(); List filters2=new ArrayList(); - filters2.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii")); + //filters2.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii")); filters2.add(new Filter("serviceClass", "DataAccess")); filters2.add(new Filter("serviceName", "CkanConnector")); - simpleFilter2.setFilters(filters2); + simpleFilter2.setFiltersValue(filters2); filterPackageQuota1.add(simpleFilter2); - Filters simpleFilter3 = new Filters(); + FiltersValue simpleFilter3 = new FiltersValue(); List filters3=new ArrayList(); - filters3.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii")); + //filters3.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii")); filters3.add(new Filter("serviceClass", "VREManagement")); - simpleFilter3.setFilters(filters3); + simpleFilter3.setFiltersValue(filters3); filterPackageQuota1.add(simpleFilter3); - TotalFilters totalfilter1=new TotalFilters(); - totalfilter1.setTotalFilters(filterPackageQuota1); + UsageValue totalfilter1=new UsageServiceValue("gcube","lucio.lelii",AggregatedServiceUsageRecord.class,temporalConstraint,filterPackageQuota1); + //totalfilter1.setTotalFilters(filterPackageQuota1); + + + + + + /*ask quota for alessandro pieve + *STORAGE + **/ + List filterPackageQuota2 =new ArrayList(); + FiltersValue simpleFilter4 = new FiltersValue(); + List filters4=new ArrayList(); + filters4.add(new Filter(AggregatedStorageUsageRecord.CONSUMER_ID, "alessandro.pieve")); + + simpleFilter4.setFiltersValue(filters4); + + filterPackageQuota2.add(simpleFilter4); + + //TotalFilters totalfilter2=new TotalFilters("alessandro.pieve",AggregatedStorageUsageRecord.class,temporalConstraint,filterPackageQuota2); + + + List filtersList=new ArrayList(); + //UsageValue totalfilter2=new UsageStorageValue("alessandro.pieve",AggregatedStorageUsageRecord.class,temporalConstraint); + + UsageValue totalfilter2=new UsageStorageValue(context,"alessandro.pieve",AggregatedStorageUsageRecord.class); + + + List listTotalFilter=new ArrayList(); +// listTotalFilter.add(totalfilter); +listTotalFilter.add(totalfilter1); + + listTotalFilter.add(totalfilter2); - List listTotalFilter=new ArrayList(); - listTotalFilter.add(totalfilter); - listTotalFilter.add(totalfilter1); logger.info("filterPackageQuota:"+listTotalFilter); - List filterValue = - accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(AggregatedServiceUsageRecord.class, temporalConstraint, listTotalFilter); + List filterValue = + accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(listTotalFilter); logger.info("result:"+filterValue.toString()); } @@ -447,11 +477,8 @@ public class AccountingPersistenceQueryCouchBaseTest { startTime.set(2016, Calendar.SEPTEMBER, 1); Calendar endTime = Calendar.getInstance(); endTime.set(2016, Calendar.OCTOBER, 20,23,59); - List filters = new ArrayList(); //filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); - - TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); @@ -460,27 +487,18 @@ public class AccountingPersistenceQueryCouchBaseTest { AggregatedServiceUsageRecord.class; List context=new ArrayList(); - context.add("/gcube"); context.add("/gcube/devNext"); context.add("/gcube/devNext/nextNext"); - context.add("/gcube/devsec"); context.add("/gcube/devsec/devVRE"); - - context.add("/gcube/devsec/preVRE"); - context.add("/gcube/preprod/preVRE"); context.add("/gcube/preprod"); - - SortedMap> setContext = accountingPersistenceQueryCouchBase.getContextTimeSeries( clz, temporalConstraint, filters, context); - - logger.debug("Result final{}", setContext); }