gCubeSystem
/
accounting-analytics-persistence-couchbase
Archived
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics-persistence-couchbase@133190 82a268e6-3cf1-43bd-a215-b396298e98cf

Browse Source
feature/19115
Alessandro Pieve 8 years ago
parent 75b9d24451
commit 882d1e116e
2 changed files with 232 additions and 11 deletions
Show Stats Download Patch File Download Diff File

155
src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java
Unescape Escape View File

@ -30,6 +30,7 @@ import org.gcube.accounting.analytics.NumberedFilter;
import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.TemporalConstraint.CalendarEnum;
import org.gcube.accounting.analytics.TotalFilters;
import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException;
import org.gcube.accounting.analytics.exception.ValueException;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
@ -42,7 +43,6 @@ import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.records.RecordUtility;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
@ -1061,4 +1061,157 @@ AccountingPersistenceBackendQuery {
/**
*
*/
@Override
public List<TotalFilters> getUsageValueQuotaTotal(
Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<TotalFilters> listUsage)
throws Exception {
String currentScope = ScopeProvider.instance.get();
AggregationMode aggregationMode = temporalConstraint
.getAggregationMode();
JsonArray temporalStartKey = getRangeKey(
temporalConstraint.getStartTime(),
aggregationMode, false, false);
JsonArray temporalEndKey = getRangeKey(
temporalConstraint.getEndTime(),
aggregationMode, false, false);
String designDocId = getDesignDocId(clz);
for (TotalFilters totalfilters:listUsage){
Double totalQuota=0.00;
for (Filters singleFilter:totalfilters.getTotalFilters()){
String viewNameTmp=null;
JsonArray startKeyTmp=JsonArray.create();
startKeyTmp.add(currentScope);
JsonArray endKeyTmp=JsonArray.create();
endKeyTmp.add(currentScope);
int groupLevelTmp= 2;
for (Filter filter:singleFilter.getFilters()){
if (groupLevelTmp==2)
viewNameTmp=filter.getKey();
else
viewNameTmp=viewNameTmp+"__"+filter.getKey();
startKeyTmp.add(filter.getValue());
endKeyTmp.add(filter.getValue());
groupLevelTmp++;
}
for (Object temporal: temporalStartKey.toList()){
if (!temporal.toString().isEmpty())
startKeyTmp.add(temporal);
}
int count =1;
for (Object temporal: temporalEndKey.toList()){
if (!temporal.toString().isEmpty()){
//couchbase exclude last value
if (count==temporalEndKey.size())
temporal=(int)temporal+1;
endKeyTmp.add(temporal);
}
count++;
}
ViewQuery query = ViewQuery.from(designDocId, viewNameTmp);
query.inclusiveEnd();
query.groupLevel(groupLevelTmp);
query.startKey(startKeyTmp);
query.endKey(endKeyTmp);
query.descending(false);
logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, "
+ "Group Level : {}, Start Key : {}, End Key : {},"
+ "temporalStartKey :{}, temporalEndKey :{}",
clz.getSimpleName(),designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp,temporalStartKey.toString(), temporalEndKey.toString());
ViewResult viewResult;
try {
//execute query in a specify bucket
viewResult = connectionMap.get(clz.getSimpleName()).query(query);
} catch (Exception e) {
logger.error(e.getLocalizedMessage());
throw e;
}
Map<String, Float> map = new HashMap<String, Float>();
for (ViewRow row : viewResult) {
JsonObject jsnobject = (JsonObject) row.value();
JSONObject objJson = new JSONObject(jsnobject.toString());
Iterator<?> iterateJosn = objJson.keys();
while( iterateJosn.hasNext() ) {
String key = (String)iterateJosn.next();
Float valuetmp=Float.parseFloat(objJson.get(key).toString());
if (key.equals("operationCount") || key.equals("dataVolume")){
if (map.containsKey(key)) {
map.put(key, valuetmp + map.get(key));
// TODO verify a better method
if (designDocId.equals("StorageUsageRecord")){
if (key.equals("dataVolume")){
singleFilter.setOrderingProperty(key);
singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue());
totalQuota+=singleFilter.getD()+valuetmp.doubleValue();
}
}
else{
singleFilter.setOrderingProperty(key);
singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue());
totalQuota+=singleFilter.getD()+valuetmp.doubleValue();
}
}
else{
map.put(key, valuetmp);
// TODO verify a better method
if (designDocId.equals("StorageUsageRecord")){
if (key.equals("dataVolume")){
singleFilter.setOrderingProperty(key);
singleFilter.setD(valuetmp.doubleValue());
totalQuota+=valuetmp.doubleValue();
}
}
else{
singleFilter.setOrderingProperty(key);
singleFilter.setD(valuetmp.doubleValue());
totalQuota+=valuetmp.doubleValue();
}
}
}
}
}
}
totalfilters.setD(totalQuota);
}
return listUsage;
}
}

88
src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java
Unescape Escape View File

@ -18,15 +18,14 @@ import org.gcube.accounting.analytics.Filters;
import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.NumberedFilter;
import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TotalFilters;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
import org.gcube.common.scope.api.ScopeProvider;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.After;
@ -217,22 +216,91 @@ public class AccountingPersistenceQueryCouchBaseTest {
simpleFilter1.setFilters(filters1);
filterPackageQuota.add(simpleFilter1);
logger.info("filterPackageQuota:"+filterPackageQuota);
List<Filters> filterValue =
accountingPersistenceQueryCouchBase.getUsageValueQuota(AggregatedServiceUsageRecord.class, temporalConstraint, filterPackageQuota);
logger.info("result:"+filterValue.toString());
}
}
@Test
public void getUsageValueQuotaTotal() throws Exception{
Calendar startTime = Calendar.getInstance();
startTime.set(2015, Calendar.MAY, 1);
Calendar endTime = Calendar.getInstance();
ScopeProvider.instance.set("/gcube");
TemporalConstraint temporalConstraint =
new TemporalConstraint(startTime.getTimeInMillis(),
endTime.getTimeInMillis(), AggregationMode.DAILY);
/*ask quota for user alessandro pieve*/
List<Filters> filterPackageQuota =new ArrayList<Filters>();
Filters simpleFilter = new Filters();
List<Filter> filters=new ArrayList<Filter>();
filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
filters.add(new Filter("serviceClass", "DataAccess"));
filters.add(new Filter("serviceName", "CkanConnector"));
simpleFilter.setFilters(filters);
filterPackageQuota.add(simpleFilter);
Filters simpleFilter1 = new Filters();
List<Filter> filters1=new ArrayList<Filter>();
filters1.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
filters1.add(new Filter("serviceClass", "VREManagement"));
simpleFilter1.setFilters(filters1);
filterPackageQuota.add(simpleFilter1);
TotalFilters totalfilter=new TotalFilters();
totalfilter.setTotalFilters(filterPackageQuota);
/*ask quota for user lucio lelii*/
List<Filters> filterPackageQuota1 =new ArrayList<Filters>();
Filters simpleFilter2 = new Filters();
List<Filter> filters2=new ArrayList<Filter>();
filters2.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii"));
filters2.add(new Filter("serviceClass", "DataAccess"));
filters2.add(new Filter("serviceName", "CkanConnector"));
simpleFilter2.setFilters(filters2);
filterPackageQuota1.add(simpleFilter2);
Filters simpleFilter3 = new Filters();
List<Filter> filters3=new ArrayList<Filter>();
filters3.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii"));
filters3.add(new Filter("serviceClass", "VREManagement"));
simpleFilter3.setFilters(filters3);
filterPackageQuota1.add(simpleFilter3);
TotalFilters totalfilter1=new TotalFilters();
totalfilter1.setTotalFilters(filterPackageQuota1);
List<TotalFilters> listTotalFilter=new ArrayList<TotalFilters>();
listTotalFilter.add(totalfilter);
listTotalFilter.add(totalfilter1);
logger.info("filterPackageQuota:"+listTotalFilter);
List<TotalFilters> filterValue =
accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(AggregatedServiceUsageRecord.class, temporalConstraint, listTotalFilter);
logger.info("result:"+filterValue.toString());
}
@Test
public void getQuerableKeyJob() throws Exception{
SortedSet<String> keys;