git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics-persistence-couchbase@133190 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
75b9d24451
commit
882d1e116e
|
@ -30,6 +30,7 @@ import org.gcube.accounting.analytics.NumberedFilter;
|
||||||
import org.gcube.accounting.analytics.TemporalConstraint;
|
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||||
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||||
import org.gcube.accounting.analytics.TemporalConstraint.CalendarEnum;
|
import org.gcube.accounting.analytics.TemporalConstraint.CalendarEnum;
|
||||||
|
import org.gcube.accounting.analytics.TotalFilters;
|
||||||
import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException;
|
import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException;
|
||||||
import org.gcube.accounting.analytics.exception.ValueException;
|
import org.gcube.accounting.analytics.exception.ValueException;
|
||||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
|
||||||
|
@ -42,7 +43,6 @@ import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.gcube.documentstore.records.AggregatedRecord;
|
import org.gcube.documentstore.records.AggregatedRecord;
|
||||||
import org.gcube.documentstore.records.Record;
|
import org.gcube.documentstore.records.Record;
|
||||||
import org.gcube.documentstore.records.RecordUtility;
|
import org.gcube.documentstore.records.RecordUtility;
|
||||||
import org.json.JSONArray;
|
|
||||||
import org.json.JSONException;
|
import org.json.JSONException;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
|
@ -1061,4 +1061,157 @@ AccountingPersistenceBackendQuery {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public List<TotalFilters> getUsageValueQuotaTotal(
|
||||||
|
Class<? extends AggregatedRecord<?, ?>> clz,
|
||||||
|
TemporalConstraint temporalConstraint, List<TotalFilters> listUsage)
|
||||||
|
throws Exception {
|
||||||
|
|
||||||
|
|
||||||
|
String currentScope = ScopeProvider.instance.get();
|
||||||
|
AggregationMode aggregationMode = temporalConstraint
|
||||||
|
.getAggregationMode();
|
||||||
|
|
||||||
|
JsonArray temporalStartKey = getRangeKey(
|
||||||
|
temporalConstraint.getStartTime(),
|
||||||
|
aggregationMode, false, false);
|
||||||
|
|
||||||
|
JsonArray temporalEndKey = getRangeKey(
|
||||||
|
temporalConstraint.getEndTime(),
|
||||||
|
aggregationMode, false, false);
|
||||||
|
|
||||||
|
String designDocId = getDesignDocId(clz);
|
||||||
|
for (TotalFilters totalfilters:listUsage){
|
||||||
|
|
||||||
|
Double totalQuota=0.00;
|
||||||
|
|
||||||
|
for (Filters singleFilter:totalfilters.getTotalFilters()){
|
||||||
|
String viewNameTmp=null;
|
||||||
|
|
||||||
|
JsonArray startKeyTmp=JsonArray.create();
|
||||||
|
startKeyTmp.add(currentScope);
|
||||||
|
|
||||||
|
JsonArray endKeyTmp=JsonArray.create();
|
||||||
|
endKeyTmp.add(currentScope);
|
||||||
|
|
||||||
|
int groupLevelTmp= 2;
|
||||||
|
|
||||||
|
for (Filter filter:singleFilter.getFilters()){
|
||||||
|
if (groupLevelTmp==2)
|
||||||
|
viewNameTmp=filter.getKey();
|
||||||
|
else
|
||||||
|
viewNameTmp=viewNameTmp+"__"+filter.getKey();
|
||||||
|
startKeyTmp.add(filter.getValue());
|
||||||
|
endKeyTmp.add(filter.getValue());
|
||||||
|
groupLevelTmp++;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Object temporal: temporalStartKey.toList()){
|
||||||
|
if (!temporal.toString().isEmpty())
|
||||||
|
startKeyTmp.add(temporal);
|
||||||
|
}
|
||||||
|
int count =1;
|
||||||
|
for (Object temporal: temporalEndKey.toList()){
|
||||||
|
if (!temporal.toString().isEmpty()){
|
||||||
|
//couchbase exclude last value
|
||||||
|
if (count==temporalEndKey.size())
|
||||||
|
temporal=(int)temporal+1;
|
||||||
|
endKeyTmp.add(temporal);
|
||||||
|
}
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
ViewQuery query = ViewQuery.from(designDocId, viewNameTmp);
|
||||||
|
query.inclusiveEnd();
|
||||||
|
query.groupLevel(groupLevelTmp);
|
||||||
|
query.startKey(startKeyTmp);
|
||||||
|
query.endKey(endKeyTmp);
|
||||||
|
query.descending(false);
|
||||||
|
|
||||||
|
logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, "
|
||||||
|
+ "Group Level : {}, Start Key : {}, End Key : {},"
|
||||||
|
+ "temporalStartKey :{}, temporalEndKey :{}",
|
||||||
|
clz.getSimpleName(),designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp,temporalStartKey.toString(), temporalEndKey.toString());
|
||||||
|
|
||||||
|
ViewResult viewResult;
|
||||||
|
try {
|
||||||
|
//execute query in a specify bucket
|
||||||
|
viewResult = connectionMap.get(clz.getSimpleName()).query(query);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, Float> map = new HashMap<String, Float>();
|
||||||
|
|
||||||
|
for (ViewRow row : viewResult) {
|
||||||
|
|
||||||
|
JsonObject jsnobject = (JsonObject) row.value();
|
||||||
|
JSONObject objJson = new JSONObject(jsnobject.toString());
|
||||||
|
|
||||||
|
Iterator<?> iterateJosn = objJson.keys();
|
||||||
|
while( iterateJosn.hasNext() ) {
|
||||||
|
String key = (String)iterateJosn.next();
|
||||||
|
Float valuetmp=Float.parseFloat(objJson.get(key).toString());
|
||||||
|
|
||||||
|
if (key.equals("operationCount") || key.equals("dataVolume")){
|
||||||
|
if (map.containsKey(key)) {
|
||||||
|
map.put(key, valuetmp + map.get(key));
|
||||||
|
|
||||||
|
// TODO verify a better method
|
||||||
|
if (designDocId.equals("StorageUsageRecord")){
|
||||||
|
if (key.equals("dataVolume")){
|
||||||
|
singleFilter.setOrderingProperty(key);
|
||||||
|
singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue());
|
||||||
|
totalQuota+=singleFilter.getD()+valuetmp.doubleValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
singleFilter.setOrderingProperty(key);
|
||||||
|
singleFilter.setD(singleFilter.getD()+valuetmp.doubleValue());
|
||||||
|
totalQuota+=singleFilter.getD()+valuetmp.doubleValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
map.put(key, valuetmp);
|
||||||
|
|
||||||
|
// TODO verify a better method
|
||||||
|
if (designDocId.equals("StorageUsageRecord")){
|
||||||
|
if (key.equals("dataVolume")){
|
||||||
|
singleFilter.setOrderingProperty(key);
|
||||||
|
singleFilter.setD(valuetmp.doubleValue());
|
||||||
|
totalQuota+=valuetmp.doubleValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
singleFilter.setOrderingProperty(key);
|
||||||
|
singleFilter.setD(valuetmp.doubleValue());
|
||||||
|
totalQuota+=valuetmp.doubleValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
totalfilters.setD(totalQuota);
|
||||||
|
|
||||||
|
}
|
||||||
|
return listUsage;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,15 +18,14 @@ import org.gcube.accounting.analytics.Filters;
|
||||||
import org.gcube.accounting.analytics.Info;
|
import org.gcube.accounting.analytics.Info;
|
||||||
import org.gcube.accounting.analytics.NumberedFilter;
|
import org.gcube.accounting.analytics.NumberedFilter;
|
||||||
import org.gcube.accounting.analytics.TemporalConstraint;
|
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||||
|
import org.gcube.accounting.analytics.TotalFilters;
|
||||||
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
|
||||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||||
import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord;
|
import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord;
|
||||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||||
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
|
|
||||||
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
|
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.json.JSONArray;
|
|
||||||
import org.json.JSONException;
|
import org.json.JSONException;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
|
@ -217,22 +216,91 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
simpleFilter1.setFilters(filters1);
|
simpleFilter1.setFilters(filters1);
|
||||||
filterPackageQuota.add(simpleFilter1);
|
filterPackageQuota.add(simpleFilter1);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
logger.info("filterPackageQuota:"+filterPackageQuota);
|
logger.info("filterPackageQuota:"+filterPackageQuota);
|
||||||
List<Filters> filterValue =
|
List<Filters> filterValue =
|
||||||
accountingPersistenceQueryCouchBase.getUsageValueQuota(AggregatedServiceUsageRecord.class, temporalConstraint, filterPackageQuota);
|
accountingPersistenceQueryCouchBase.getUsageValueQuota(AggregatedServiceUsageRecord.class, temporalConstraint, filterPackageQuota);
|
||||||
|
|
||||||
|
|
||||||
logger.info("result:"+filterValue.toString());
|
logger.info("result:"+filterValue.toString());
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void getUsageValueQuotaTotal() throws Exception{
|
||||||
|
Calendar startTime = Calendar.getInstance();
|
||||||
|
startTime.set(2015, Calendar.MAY, 1);
|
||||||
|
Calendar endTime = Calendar.getInstance();
|
||||||
|
|
||||||
|
|
||||||
|
ScopeProvider.instance.set("/gcube");
|
||||||
|
|
||||||
|
TemporalConstraint temporalConstraint =
|
||||||
|
new TemporalConstraint(startTime.getTimeInMillis(),
|
||||||
|
endTime.getTimeInMillis(), AggregationMode.DAILY);
|
||||||
|
|
||||||
|
|
||||||
|
/*ask quota for user alessandro pieve*/
|
||||||
|
List<Filters> filterPackageQuota =new ArrayList<Filters>();
|
||||||
|
|
||||||
|
Filters simpleFilter = new Filters();
|
||||||
|
List<Filter> filters=new ArrayList<Filter>();
|
||||||
|
filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
|
||||||
|
filters.add(new Filter("serviceClass", "DataAccess"));
|
||||||
|
filters.add(new Filter("serviceName", "CkanConnector"));
|
||||||
|
simpleFilter.setFilters(filters);
|
||||||
|
|
||||||
|
filterPackageQuota.add(simpleFilter);
|
||||||
|
|
||||||
|
|
||||||
|
Filters simpleFilter1 = new Filters();
|
||||||
|
List<Filter> filters1=new ArrayList<Filter>();
|
||||||
|
filters1.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
|
||||||
|
filters1.add(new Filter("serviceClass", "VREManagement"));
|
||||||
|
simpleFilter1.setFilters(filters1);
|
||||||
|
filterPackageQuota.add(simpleFilter1);
|
||||||
|
|
||||||
|
TotalFilters totalfilter=new TotalFilters();
|
||||||
|
totalfilter.setTotalFilters(filterPackageQuota);
|
||||||
|
|
||||||
|
|
||||||
|
/*ask quota for user lucio lelii*/
|
||||||
|
List<Filters> filterPackageQuota1 =new ArrayList<Filters>();
|
||||||
|
|
||||||
|
Filters simpleFilter2 = new Filters();
|
||||||
|
List<Filter> filters2=new ArrayList<Filter>();
|
||||||
|
filters2.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii"));
|
||||||
|
filters2.add(new Filter("serviceClass", "DataAccess"));
|
||||||
|
filters2.add(new Filter("serviceName", "CkanConnector"));
|
||||||
|
simpleFilter2.setFilters(filters2);
|
||||||
|
|
||||||
|
filterPackageQuota1.add(simpleFilter2);
|
||||||
|
|
||||||
|
|
||||||
|
Filters simpleFilter3 = new Filters();
|
||||||
|
List<Filter> filters3=new ArrayList<Filter>();
|
||||||
|
filters3.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "lucio.lelii"));
|
||||||
|
filters3.add(new Filter("serviceClass", "VREManagement"));
|
||||||
|
simpleFilter3.setFilters(filters3);
|
||||||
|
filterPackageQuota1.add(simpleFilter3);
|
||||||
|
|
||||||
|
TotalFilters totalfilter1=new TotalFilters();
|
||||||
|
totalfilter1.setTotalFilters(filterPackageQuota1);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
List<TotalFilters> listTotalFilter=new ArrayList<TotalFilters>();
|
||||||
|
listTotalFilter.add(totalfilter);
|
||||||
|
listTotalFilter.add(totalfilter1);
|
||||||
|
logger.info("filterPackageQuota:"+listTotalFilter);
|
||||||
|
List<TotalFilters> filterValue =
|
||||||
|
accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(AggregatedServiceUsageRecord.class, temporalConstraint, listTotalFilter);
|
||||||
|
logger.info("result:"+filterValue.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void getQuerableKeyJob() throws Exception{
|
public void getQuerableKeyJob() throws Exception{
|
||||||
SortedSet<String> keys;
|
SortedSet<String> keys;
|
||||||
|
|
Reference in New Issue