From 882d1e116eb37af06f7e85f0273e8a1c060fa2f6 Mon Sep 17 00:00:00 2001 From: Alessandro Pieve Date: Thu, 13 Oct 2016 16:18:38 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics-persistence-couchbase@133190 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../AccountingPersistenceQueryCouchBase.java | 155 +++++++++++++++++- ...countingPersistenceQueryCouchBaseTest.java | 90 ++++++++-- 2 files changed, 233 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java b/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java index 009d9ab..fa5bcb0 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java @@ -30,6 +30,7 @@ import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.TemporalConstraint.CalendarEnum; +import org.gcube.accounting.analytics.TotalFilters; import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException; import org.gcube.accounting.analytics.exception.ValueException; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; @@ -42,7 +43,6 @@ import org.gcube.common.scope.api.ScopeProvider; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.RecordUtility; -import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; @@ -1061,4 +1061,157 @@ AccountingPersistenceBackendQuery { + /** + * + */ + @Override + public List getUsageValueQuotaTotal( + Class> clz, + TemporalConstraint temporalConstraint, List listUsage) + throws Exception { + + + String currentScope = ScopeProvider.instance.get(); + AggregationMode aggregationMode = temporalConstraint + .getAggregationMode(); + + JsonArray temporalStartKey = getRangeKey( + temporalConstraint.getStartTime(), + aggregationMode, false, false); + + JsonArray temporalEndKey = getRangeKey( + temporalConstraint.getEndTime(), + aggregationMode, false, false); + + String designDocId = getDesignDocId(clz); + for (TotalFilters totalfilters:listUsage){ + + Double totalQuota=0.00; + + for (Filters singleFilter:totalfilters.getTotalFilters()){ + String viewNameTmp=null; + + JsonArray startKeyTmp=JsonArray.create(); + startKeyTmp.add(currentScope); + + JsonArray endKeyTmp=JsonArray.create(); + endKeyTmp.add(currentScope); + + int groupLevelTmp= 2; + + for (Filter filter:singleFilter.getFilters()){ + if (groupLevelTmp==2) + viewNameTmp=filter.getKey(); + else + viewNameTmp=viewNameTmp+"__"+filter.getKey(); + startKeyTmp.add(filter.getValue()); + endKeyTmp.add(filter.getValue()); + groupLevelTmp++; + + } + + for (Object temporal: temporalStartKey.toList()){ + if (!temporal.toString().isEmpty()) + startKeyTmp.add(temporal); + } + int count =1; + for (Object temporal: temporalEndKey.toList()){ + if (!temporal.toString().isEmpty()){ + //couchbase exclude last value + if (count==temporalEndKey.size()) + temporal=(int)temporal+1; + endKeyTmp.add(temporal); + } + count++; + } + + ViewQuery query = ViewQuery.from(designDocId, viewNameTmp); + query.inclusiveEnd(); + query.groupLevel(groupLevelTmp); + query.startKey(startKeyTmp); + query.endKey(endKeyTmp); + query.descending(false); + + logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, " + + "Group Level : {}, Start Key : {}, End Key : {}," + + "temporalStartKey :{}, temporalEndKey :{}", + clz.getSimpleName(),designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp,temporalStartKey.toString(), temporalEndKey.toString()); + + ViewResult viewResult; + try { + //execute query in a specify bucket + viewResult = connectionMap.get(clz.getSimpleName()).query(query); + + } catch (Exception e) { + logger.error(e.getLocalizedMessage()); + throw e; + } + + Map map = new HashMap(); + + for (ViewRow row : viewResult) { + + JsonObject jsnobject = (JsonObject) row.value(); + JSONObject objJson = new JSONObject(jsnobject.toString()); + + Iterator iterateJosn = objJson.keys(); + while( iterateJosn.hasNext() ) { + String key = (String)iterateJosn.next(); + Float valuetmp=Float.parseFloat(objJson.get(key).toString()); + + if (key.equals("operationCount") || key.equals("dataVolume")){ + if (map.containsKey(key)) { + map.put(key, valuetmp + map.get(key)); + + // TODO verify a better method + if (designDocId.equals("StorageUsageRecord")){ + if (key.equals("dataVolume")){ + singleFilter.setOrderingProperty(key); + singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue()); + totalQuota+=singleFilter.getD()+valuetmp.doubleValue(); + } + } + else{ + singleFilter.setOrderingProperty(key); + singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue()); + totalQuota+=singleFilter.getD()+valuetmp.doubleValue(); + } + + } + else{ + map.put(key, valuetmp); + + // TODO verify a better method + if (designDocId.equals("StorageUsageRecord")){ + if (key.equals("dataVolume")){ + singleFilter.setOrderingProperty(key); + singleFilter.setD(valuetmp.doubleValue()); + totalQuota+=valuetmp.doubleValue(); + } + } + else{ + singleFilter.setOrderingProperty(key); + singleFilter.setD(valuetmp.doubleValue()); + totalQuota+=valuetmp.doubleValue(); + } + + } + + } + } + } + } + totalfilters.setD(totalQuota); + + } + return listUsage; + + + + + + } + + + } diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java index 062490d..940529e 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java @@ -18,15 +18,14 @@ import org.gcube.accounting.analytics.Filters; import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.TemporalConstraint; +import org.gcube.accounting.analytics.TotalFilters; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; -import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord; import org.gcube.common.scope.api.ScopeProvider; -import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.junit.After; @@ -217,22 +216,91 @@ public class AccountingPersistenceQueryCouchBaseTest { simpleFilter1.setFilters(filters1); filterPackageQuota.add(simpleFilter1); - - - - logger.info("filterPackageQuota:"+filterPackageQuota); List filterValue = accountingPersistenceQueryCouchBase.getUsageValueQuota(AggregatedServiceUsageRecord.class, temporalConstraint, filterPackageQuota); - - logger.info("result:"+filterValue.toString()); - - - } + + @Test + public void getUsageValueQuotaTotal() throws Exception{ + Calendar startTime = Calendar.getInstance(); + startTime.set(2015, Calendar.MAY, 1); + Calendar endTime = Calendar.getInstance(); + + + ScopeProvider.instance.set("/gcube"); + + TemporalConstraint temporalConstraint = + new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + + + /*ask quota for user alessandro pieve*/ + List filterPackageQuota =new ArrayList(); + + Filters simpleFilter = new Filters(); + List filters=new ArrayList(); + filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); + filters.add(new Filter("serviceClass", "DataAccess")); + filters.add(new Filter("serviceName", "CkanConnector")); + simpleFilter.setFilters(filters); + + filterPackageQuota.add(simpleFilter); + + + Filters simpleFilter1 = new Filters(); + List filters1=new ArrayList(); + filters1.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); + filters1.add(new Filter("serviceClass", "VREManagement")); + simpleFilter1.setFilters(filters1); + filterPackageQuota.add(simpleFilter1); + + TotalFilters totalfilter=new TotalFilters(); + totalfilter.setTotalFilters(filterPackageQuota); + + + /*ask quota for user lucio lelii*/ + List filterPackageQuota1 =new ArrayList(); + + Filters simpleFilter2 = new Filters(); + List filters2=new ArrayList(); + filters2.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii")); + filters2.add(new Filter("serviceClass", "DataAccess")); + filters2.add(new Filter("serviceName", "CkanConnector")); + simpleFilter2.setFilters(filters2); + + filterPackageQuota1.add(simpleFilter2); + + + Filters simpleFilter3 = new Filters(); + List filters3=new ArrayList(); + filters3.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii")); + filters3.add(new Filter("serviceClass", "VREManagement")); + simpleFilter3.setFilters(filters3); + filterPackageQuota1.add(simpleFilter3); + + TotalFilters totalfilter1=new TotalFilters(); + totalfilter1.setTotalFilters(filterPackageQuota1); + + + + + List listTotalFilter=new ArrayList(); + listTotalFilter.add(totalfilter); + listTotalFilter.add(totalfilter1); + logger.info("filterPackageQuota:"+listTotalFilter); + List filterValue = + accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(AggregatedServiceUsageRecord.class, temporalConstraint, listTotalFilter); + logger.info("result:"+filterValue.toString()); + } + + + + + @Test public void getQuerableKeyJob() throws Exception{ SortedSet keys;