change method for calculate quota (considered Mongo and Rstudio)
add method getSpaceProvidersIds (for list providers usage portlet accounting) add method getSpaceTimeSeries (for calculate space time series) git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics-persistence-couchbase@148334 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
3d702a9b3a
commit
5039ef1b11
|
@ -0,0 +1,2 @@
|
||||||
|
disabled=06target
|
||||||
|
eclipse.preferences.version=1
|
2
pom.xml
2
pom.xml
|
@ -8,7 +8,7 @@
|
||||||
</parent>
|
</parent>
|
||||||
<groupId>org.gcube.accounting</groupId>
|
<groupId>org.gcube.accounting</groupId>
|
||||||
<artifactId>accounting-analytics-persistence-couchbase</artifactId>
|
<artifactId>accounting-analytics-persistence-couchbase</artifactId>
|
||||||
<version>1.3.0-SNAPSHOT</version>
|
<version>1.4.0-SNAPSHOT</version>
|
||||||
<name>Accounting Analytics Persistence CouchBase</name>
|
<name>Accounting Analytics Persistence CouchBase</name>
|
||||||
<description>Accounting Analytics Persistence CouchBase Implementation</description>
|
<description>Accounting Analytics Persistence CouchBase Implementation</description>
|
||||||
|
|
||||||
|
|
|
@ -46,6 +46,7 @@ import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.gcube.documentstore.records.AggregatedRecord;
|
import org.gcube.documentstore.records.AggregatedRecord;
|
||||||
import org.gcube.documentstore.records.Record;
|
import org.gcube.documentstore.records.Record;
|
||||||
import org.gcube.documentstore.records.RecordUtility;
|
import org.gcube.documentstore.records.RecordUtility;
|
||||||
|
import org.json.JSONArray;
|
||||||
import org.json.JSONException;
|
import org.json.JSONException;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
|
@ -55,6 +56,7 @@ import com.couchbase.client.java.Bucket;
|
||||||
import com.couchbase.client.java.Cluster;
|
import com.couchbase.client.java.Cluster;
|
||||||
import com.couchbase.client.java.CouchbaseCluster;
|
import com.couchbase.client.java.CouchbaseCluster;
|
||||||
import com.couchbase.client.java.bucket.BucketManager;
|
import com.couchbase.client.java.bucket.BucketManager;
|
||||||
|
import com.couchbase.client.java.document.JsonDocument;
|
||||||
import com.couchbase.client.java.document.json.JsonArray;
|
import com.couchbase.client.java.document.json.JsonArray;
|
||||||
import com.couchbase.client.java.document.json.JsonObject;
|
import com.couchbase.client.java.document.json.JsonObject;
|
||||||
import com.couchbase.client.java.env.CouchbaseEnvironment;
|
import com.couchbase.client.java.env.CouchbaseEnvironment;
|
||||||
|
@ -149,12 +151,9 @@ AccountingPersistenceBackendQuery {
|
||||||
|
|
||||||
String url = configuration.getProperty(URL_PROPERTY_KEY);
|
String url = configuration.getProperty(URL_PROPERTY_KEY);
|
||||||
String password = configuration.getProperty(PASSWORD_PROPERTY_KEY);
|
String password = configuration.getProperty(PASSWORD_PROPERTY_KEY);
|
||||||
|
|
||||||
cluster = CouchbaseCluster.create(ENV, url);
|
cluster = CouchbaseCluster.create(ENV, url);
|
||||||
bucketNameStorage = configuration.getProperty(BUCKET_STORAGE_NAME_PROPERTY_KEY);
|
bucketNameStorage = configuration.getProperty(BUCKET_STORAGE_NAME_PROPERTY_KEY);
|
||||||
|
|
||||||
bucketNameStorageStatus = configuration.getProperty(BUCKET_STORAGESTATUS_NAME_PROPERTY_KEY);
|
bucketNameStorageStatus = configuration.getProperty(BUCKET_STORAGESTATUS_NAME_PROPERTY_KEY);
|
||||||
|
|
||||||
bucketNameService = configuration.getProperty(BUCKET_SERVICE_NAME_PROPERTY_KEY);
|
bucketNameService = configuration.getProperty(BUCKET_SERVICE_NAME_PROPERTY_KEY);
|
||||||
bucketNameJob = configuration.getProperty(BUCKET_JOB_NAME_PROPERTY_KEY);
|
bucketNameJob = configuration.getProperty(BUCKET_JOB_NAME_PROPERTY_KEY);
|
||||||
bucketNamePortlet = configuration.getProperty(BUCKET_PORTLET_NAME_PROPERTY_KEY);
|
bucketNamePortlet = configuration.getProperty(BUCKET_PORTLET_NAME_PROPERTY_KEY);
|
||||||
|
@ -162,14 +161,13 @@ AccountingPersistenceBackendQuery {
|
||||||
|
|
||||||
connectionMap = new HashMap<String, Bucket>();
|
connectionMap = new HashMap<String, Bucket>();
|
||||||
|
|
||||||
|
//bucketStorage = cluster.openBucket(bucketNameStorage, password);
|
||||||
bucketStorage = cluster.openBucket(bucketNameStorage, password);
|
bucketStorage = cluster.openBucket(bucketNameStorage, password);
|
||||||
connectionMap.put(BUCKET_STORAGE_NAME_PROPERTY_KEY, bucketStorage);
|
connectionMap.put(BUCKET_STORAGE_NAME_PROPERTY_KEY, bucketStorage);
|
||||||
|
|
||||||
bucketStorageStatus = cluster.openBucket( bucketNameStorageStatus,password);
|
bucketStorageStatus = cluster.openBucket( bucketNameStorageStatus,password);
|
||||||
connectionMap.put(BUCKET_STORAGESTATUS_NAME_PROPERTY_KEY, bucketStorageStatus);
|
connectionMap.put(BUCKET_STORAGESTATUS_NAME_PROPERTY_KEY, bucketStorageStatus);
|
||||||
|
|
||||||
logger.debug("connectionMap"+connectionMap.toString());
|
|
||||||
|
|
||||||
bucketService = cluster.openBucket(bucketNameService, password);
|
bucketService = cluster.openBucket(bucketNameService, password);
|
||||||
connectionMap.put(BUCKET_SERVICE_NAME_PROPERTY_KEY, bucketService);
|
connectionMap.put(BUCKET_SERVICE_NAME_PROPERTY_KEY, bucketService);
|
||||||
|
|
||||||
|
@ -417,7 +415,6 @@ AccountingPersistenceBackendQuery {
|
||||||
|
|
||||||
if (!collection.isEmpty()){
|
if (!collection.isEmpty()){
|
||||||
reduceFunction = top;
|
reduceFunction = top;
|
||||||
//reduceFunction = null;
|
|
||||||
for (String property : collection) {
|
for (String property : collection) {
|
||||||
if (!property.equals(top)){
|
if (!property.equals(top)){
|
||||||
|
|
||||||
|
@ -466,6 +463,7 @@ AccountingPersistenceBackendQuery {
|
||||||
*Return:
|
*Return:
|
||||||
* List of service class used by alessandro in the required period
|
* List of service class used by alessandro in the required period
|
||||||
*/
|
*/
|
||||||
|
/*
|
||||||
public SortedMap<String,Integer> getListUsage(Class<? extends AggregatedRecord<?, ?>> clz,
|
public SortedMap<String,Integer> getListUsage(Class<? extends AggregatedRecord<?, ?>> clz,
|
||||||
TemporalConstraint temporalConstraint, List<Filter> filters,String context,List<String> parameters)throws Exception{
|
TemporalConstraint temporalConstraint, List<Filter> filters,String context,List<String> parameters)throws Exception{
|
||||||
//TODO
|
//TODO
|
||||||
|
@ -520,13 +518,7 @@ AccountingPersistenceBackendQuery {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/*
|
|
||||||
int scopeDateGroupLevel = aggregationMode.ordinal() + 1 + 1;
|
|
||||||
int groupLevel = scopeDateGroupLevel;
|
|
||||||
if (filters != null) {
|
|
||||||
groupLevel += keys.size();
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
for (Object temporal: temporalStartKey.toList()){
|
for (Object temporal: temporalStartKey.toList()){
|
||||||
if (!temporal.toString().isEmpty())
|
if (!temporal.toString().isEmpty())
|
||||||
startKey.add(temporal);
|
startKey.add(temporal);
|
||||||
|
@ -587,7 +579,7 @@ AccountingPersistenceBackendQuery {
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
protected SortedMap<Calendar, Info> mapReduceQuery(
|
protected SortedMap<Calendar, Info> mapReduceQuery(
|
||||||
Class<? extends AggregatedRecord<?, ?>> clz,
|
Class<? extends AggregatedRecord<?, ?>> clz,
|
||||||
|
@ -1499,7 +1491,7 @@ AccountingPersistenceBackendQuery {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* SPERIMENTAL
|
*
|
||||||
* Used for calculate a usage value for each element of list
|
* Used for calculate a usage value for each element of list
|
||||||
* (QUOTA)
|
* (QUOTA)
|
||||||
* @param listUsage
|
* @param listUsage
|
||||||
|
@ -1518,14 +1510,6 @@ AccountingPersistenceBackendQuery {
|
||||||
for (UsageValue totalFilters:listUsage){
|
for (UsageValue totalFilters:listUsage){
|
||||||
|
|
||||||
logger.debug("-----------------/ init for totalFilters");
|
logger.debug("-----------------/ init for totalFilters");
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
UsageValue totalfilter=
|
|
||||||
new UsageServiceValue(context,"lucio.lelii",AggregatedServiceUsageRecord.class,temporalConstraint,filters);
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
String currentScope = totalFilters.getContext();
|
String currentScope = totalFilters.getContext();
|
||||||
|
|
||||||
Collection<String> keys= new TreeSet<>();
|
Collection<String> keys= new TreeSet<>();
|
||||||
|
@ -1547,12 +1531,14 @@ AccountingPersistenceBackendQuery {
|
||||||
temporalConstraint =
|
temporalConstraint =
|
||||||
new TemporalConstraint(startTime.getTimeInMillis(),
|
new TemporalConstraint(startTime.getTimeInMillis(),
|
||||||
endTime.getTimeInMillis(), AggregationMode.DAILY);
|
endTime.getTimeInMillis(), AggregationMode.DAILY);
|
||||||
//if type=storage then bucket =accounting_storage_status and designDocId = quotaTotal
|
if (totalFilters instanceof UsageStorageValue){
|
||||||
//logger.trace("totalFilters.getClass().getSimpleName():{}",totalFilters.getClass().getSimpleName());
|
|
||||||
//logger.trace("totalFilters.getClz().getSimpleName():{}",totalFilters.getClz().getSimpleName());
|
// if (totalFilters.getClz().getSimpleName().equals(AggregatedStorageStatusRecord.class.getSimpleName())){
|
||||||
if (totalFilters.getClz().getSimpleName().equals(AggregatedStorageStatusRecord.class.getSimpleName())){
|
|
||||||
//TODO sarebbe possibile invece con il nome verificare che getClz e' un istanza di AggregatedStorageStatusRecord?
|
//TODO sarebbe possibile invece con il nome verificare che getClz e' un istanza di AggregatedStorageStatusRecord?
|
||||||
designDocId = "QuotaTotal";
|
//designDocId = "QuotaTotal";
|
||||||
|
|
||||||
|
//TEST
|
||||||
|
designDocId="QuotaTotalSeparated";
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1628,6 +1614,7 @@ AccountingPersistenceBackendQuery {
|
||||||
ViewQuery query = ViewQuery.from(designDocId, viewNameTmp);
|
ViewQuery query = ViewQuery.from(designDocId, viewNameTmp);
|
||||||
//if storage
|
//if storage
|
||||||
query.inclusiveEnd();
|
query.inclusiveEnd();
|
||||||
|
/*
|
||||||
if (designDocId == "QuotaTotal"){
|
if (designDocId == "QuotaTotal"){
|
||||||
|
|
||||||
query.startKey(endKeyTmp);
|
query.startKey(endKeyTmp);
|
||||||
|
@ -1638,13 +1625,13 @@ AccountingPersistenceBackendQuery {
|
||||||
|
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
|
*/
|
||||||
query.groupLevel(groupLevelTmp);
|
query.groupLevel(groupLevelTmp);
|
||||||
query.startKey(startKeyTmp);
|
query.startKey(startKeyTmp);
|
||||||
query.endKey(endKeyTmp);
|
query.endKey(endKeyTmp);
|
||||||
query.descending(false);
|
query.descending(false);
|
||||||
query.onError(OnError.STOP);
|
query.onError(OnError.STOP);
|
||||||
}
|
//}
|
||||||
|
|
||||||
logger.trace("query row:{}",query.toString());
|
logger.trace("query row:{}",query.toString());
|
||||||
ViewResult viewResult;
|
ViewResult viewResult;
|
||||||
|
@ -1667,54 +1654,33 @@ AccountingPersistenceBackendQuery {
|
||||||
while( iterateJosn.hasNext() ) {
|
while( iterateJosn.hasNext() ) {
|
||||||
String key = (String)iterateJosn.next();
|
String key = (String)iterateJosn.next();
|
||||||
|
|
||||||
Float valuetmp=Float.parseFloat(objJson.get(key).toString());
|
if (totalFilters instanceof UsageStorageValue){
|
||||||
|
JSONArray valueStorage=(JSONArray) objJson.get(key);
|
||||||
//logger.debug("totalFilters.getClass().getSimpleName():{} UsageStorageValue.class.getSimpleName():{}", totalFilters.getClass().getSimpleName(),UsageStorageValue.class.getSimpleName());
|
logger.debug("--storageUsageRecord -key:{} value:{}",key,valueStorage.get(0));
|
||||||
|
totalQuota+=Float.parseFloat(valueStorage.get(0).toString());
|
||||||
|
keyOrderingProperty="dataVolume";
|
||||||
|
}
|
||||||
|
else{
|
||||||
if (key.equals("operationCount") || key.equals("dataVolume")){
|
if (key.equals("operationCount") || key.equals("dataVolume")){
|
||||||
|
Float valuetmp=Float.parseFloat(objJson.get(key).toString());
|
||||||
|
|
||||||
if (map.containsKey(key)) {
|
if (map.containsKey(key)) {
|
||||||
map.put(key, valuetmp + map.get(key));
|
map.put(key, valuetmp + map.get(key));
|
||||||
|
|
||||||
// TODO verify a better method
|
|
||||||
if (totalFilters instanceof UsageStorageValue){
|
|
||||||
logger.debug("storageUsageRecord -designDocId:{}",designDocId);
|
|
||||||
if (key.equals("dataVolume")){
|
|
||||||
keyOrderingProperty=key;
|
|
||||||
totalQuota+=totalFilters.getD()+valuetmp.doubleValue();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else{
|
|
||||||
logger.debug("?UsageRecord -designDocId:{}",designDocId);
|
logger.debug("?UsageRecord -designDocId:{}",designDocId);
|
||||||
keyOrderingProperty=key;
|
keyOrderingProperty=key;
|
||||||
totalQuota+=totalFilters.getD()+valuetmp.doubleValue();
|
totalQuota+=totalFilters.getD()+valuetmp.doubleValue();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
|
||||||
else{
|
else{
|
||||||
map.put(key, valuetmp);
|
map.put(key, valuetmp);
|
||||||
|
|
||||||
// TODO verify a better method
|
|
||||||
if (totalFilters instanceof UsageStorageValue){
|
|
||||||
logger.debug("storageUsageRecord -designDocId:{}",designDocId);
|
|
||||||
if (key.equals("dataVolume")){
|
|
||||||
keyOrderingProperty=key;
|
|
||||||
|
|
||||||
totalQuota+=valuetmp.doubleValue();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else{
|
|
||||||
logger.debug("?UsageRecord -designDocId:{}",designDocId);
|
logger.debug("?UsageRecord -designDocId:{}",designDocId);
|
||||||
keyOrderingProperty=key;
|
keyOrderingProperty=key;
|
||||||
|
|
||||||
totalQuota+=valuetmp.doubleValue();
|
totalQuota+=valuetmp.doubleValue();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
//i++;
|
|
||||||
//} while (i <countFilters );
|
|
||||||
//convert usage from byte to Mb
|
//convert usage from byte to Mb
|
||||||
if (totalFilters instanceof UsageStorageValue){
|
if (totalFilters instanceof UsageStorageValue){
|
||||||
totalQuota=totalQuota/1024/1024;
|
totalQuota=totalQuota/1024/1024;
|
||||||
|
@ -1735,4 +1701,249 @@ AccountingPersistenceBackendQuery {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getRecord(String recordId, String type )
|
||||||
|
throws Exception {
|
||||||
|
|
||||||
|
try{
|
||||||
|
String recordType=null;
|
||||||
|
switch (type) {
|
||||||
|
case "service": recordType = BUCKET_SERVICE_NAME_PROPERTY_KEY;
|
||||||
|
break;
|
||||||
|
case "storage": recordType = BUCKET_STORAGE_NAME_PROPERTY_KEY;
|
||||||
|
break;
|
||||||
|
case "storageStatus": recordType = BUCKET_STORAGESTATUS_NAME_PROPERTY_KEY;
|
||||||
|
break;
|
||||||
|
case "task": recordType = BUCKET_TASK_NAME_PROPERTY_KEY;
|
||||||
|
break;
|
||||||
|
case "portlet": recordType = BUCKET_PORTLET_NAME_PROPERTY_KEY;
|
||||||
|
break;
|
||||||
|
case "job": recordType = BUCKET_JOB_NAME_PROPERTY_KEY;
|
||||||
|
break;
|
||||||
|
default: recordType = BUCKET_SERVICE_NAME_PROPERTY_KEY;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
JsonDocument recordJson=connectionMap.get(recordType).get(recordId);
|
||||||
|
|
||||||
|
return recordJson.content().toString();
|
||||||
|
}catch(Exception e){
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Used for storage status aka tab Space (into portlet accounting) and popolate a list combobox used
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public SortedSet<String> getSpaceProvidersIds() {
|
||||||
|
String currentScope = ScopeProvider.instance.get();
|
||||||
|
|
||||||
|
JsonArray startKey = JsonArray.create();
|
||||||
|
startKey.add(currentScope);
|
||||||
|
|
||||||
|
int scopeDateGroupLevel = 2;
|
||||||
|
int groupLevel = scopeDateGroupLevel;
|
||||||
|
String designDocId = "StorageStatusRecordValue";
|
||||||
|
String viewName = "providerId";
|
||||||
|
logger.trace("designDocId:{} view:{} startKey:{} groupLevel:{}",designDocId,viewName,startKey,groupLevel);
|
||||||
|
ViewQuery query = ViewQuery.from(designDocId, viewName);
|
||||||
|
|
||||||
|
query.inclusiveEnd();
|
||||||
|
query.groupLevel(groupLevel);
|
||||||
|
query.startKey(startKey);
|
||||||
|
query.descending(false);
|
||||||
|
SortedSet ret = new TreeSet();
|
||||||
|
ViewResult viewResult;
|
||||||
|
try {
|
||||||
|
//execute query in a specify bucket
|
||||||
|
viewResult = connectionMap.get("AggregatedStorageStatusRecord").query(query);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("error executing the query",e);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
for (ViewRow row : viewResult) {
|
||||||
|
String value =(String) row.value();
|
||||||
|
ret.add(value);
|
||||||
|
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* used for accounting portlet section storage status
|
||||||
|
* @param clz
|
||||||
|
* @param temporalConstraint
|
||||||
|
* @param filters
|
||||||
|
* @param context
|
||||||
|
* @return
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
|
||||||
|
public SortedMap<Filter, SortedMap<Calendar, Long>> getSpaceTimeSeries(
|
||||||
|
Class<? extends AggregatedRecord<?, ?>> clz,
|
||||||
|
TemporalConstraint temporalConstraint, List<Filter> filters,
|
||||||
|
List<String> providersId) throws Exception {
|
||||||
|
|
||||||
|
String currentScope=null;
|
||||||
|
currentScope = ScopeProvider.instance.get();
|
||||||
|
|
||||||
|
JsonArray startKey = JsonArray.create();
|
||||||
|
|
||||||
|
JsonArray endKey = JsonArray.create();
|
||||||
|
startKey.add(currentScope);
|
||||||
|
endKey.add(currentScope);
|
||||||
|
|
||||||
|
AggregationMode aggregationMode = temporalConstraint
|
||||||
|
.getAggregationMode();
|
||||||
|
|
||||||
|
JsonArray temporalStartKey = getRangeKey(
|
||||||
|
temporalConstraint.getStartTime(),
|
||||||
|
aggregationMode, false, false);
|
||||||
|
|
||||||
|
JsonArray temporalEndKey = getRangeKey(
|
||||||
|
temporalConstraint.getEndTime(),
|
||||||
|
aggregationMode, false, false);
|
||||||
|
Collection<String> keys = new TreeSet<>();
|
||||||
|
|
||||||
|
|
||||||
|
String designDocId = "StorageStatusUsage";
|
||||||
|
String viewName;
|
||||||
|
int groupLevel=5;
|
||||||
|
if (temporalConstraint.getAggregationMode().equals(AggregationMode.MONTHLY)){
|
||||||
|
groupLevel= 4;
|
||||||
|
}
|
||||||
|
viewName= temporalConstraint.getAggregationMode().name().toLowerCase();
|
||||||
|
|
||||||
|
if (filters != null && filters.size() != 0) {
|
||||||
|
// Sorting filter for call a mapreduce
|
||||||
|
Collections.sort(filters, new Comparator<Filter>() {
|
||||||
|
@Override
|
||||||
|
public int compare(Filter filter1, Filter filter2)
|
||||||
|
{
|
||||||
|
int result =filter1.getKey().compareTo(filter2.getKey());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
for (Filter filter : filters) {
|
||||||
|
String filterKey = filter.getKey();
|
||||||
|
String filterValue = filter.getValue();
|
||||||
|
|
||||||
|
if (filterKey != null && filterKey.compareTo("") != 0) {
|
||||||
|
if (filterValue != null && filterValue.compareTo("") != 0) {
|
||||||
|
if (keys.contains(filterKey)) {
|
||||||
|
throw new DuplicatedKeyFilterException(
|
||||||
|
"Only one value per Filter key is allowed");
|
||||||
|
}
|
||||||
|
startKey.add(filterValue);
|
||||||
|
endKey.add(filterValue);
|
||||||
|
keys.add(filterKey);
|
||||||
|
viewName=viewName+"_"+filterKey;
|
||||||
|
if (filterKey!="consumerId"){
|
||||||
|
groupLevel= groupLevel+1;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw new KeyException(
|
||||||
|
String.format("Invalid %s : %s",Filter.class.getSimpleName(), filter.toString()));
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
throw new ValueException(String.format("Invalid %s : %s",Filter.class.getSimpleName(), filter.toString()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (Object temporal: temporalStartKey.toList()){
|
||||||
|
if (!temporal.toString().isEmpty())
|
||||||
|
startKey.add(temporal);
|
||||||
|
}
|
||||||
|
int count =1;
|
||||||
|
for (Object temporal: temporalEndKey.toList()){
|
||||||
|
if (!temporal.toString().isEmpty()){
|
||||||
|
//couchbase exclude last value
|
||||||
|
if (count==temporalEndKey.size())
|
||||||
|
temporal=(int)temporal+1;
|
||||||
|
endKey.add(temporal);
|
||||||
|
}
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
ViewQuery query = ViewQuery.from(designDocId, viewName);
|
||||||
|
query.inclusiveEnd();
|
||||||
|
query.groupLevel(groupLevel);
|
||||||
|
query.startKey(startKey);
|
||||||
|
query.endKey(endKey);
|
||||||
|
query.descending(false);
|
||||||
|
|
||||||
|
logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, "
|
||||||
|
+ "Group Level : {}, Start Key : {}, End Key : {},"
|
||||||
|
+ "temporalStartKey :{}, temporalEndKey :{}",
|
||||||
|
clz.getSimpleName(),designDocId, viewName, groupLevel, startKey, endKey,temporalStartKey.toString(), temporalEndKey.toString());
|
||||||
|
|
||||||
|
|
||||||
|
ViewResult viewResult;
|
||||||
|
try {
|
||||||
|
//execute query in a specify bucket
|
||||||
|
viewResult = connectionMap.get(clz.getSimpleName()).query(query);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getLocalizedMessage());
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
|
||||||
|
SortedMap<Filter, SortedMap<Calendar, Long>> ret = new TreeMap<>();
|
||||||
|
|
||||||
|
for (ViewRow row : viewResult) {
|
||||||
|
|
||||||
|
JsonArray array = (JsonArray) row.key();
|
||||||
|
Calendar calendar = getCalendarFromArray(array);
|
||||||
|
|
||||||
|
JsonObject value = (JsonObject) row.value();
|
||||||
|
JSONObject obj = new JSONObject(value.toString());
|
||||||
|
|
||||||
|
//logger.trace("row: {}, value: {}, obj: {}",row.toString(),value.toString(),obj.toString());
|
||||||
|
for(Iterator iterator = obj.keys(); iterator.hasNext();) {
|
||||||
|
String key = (String) iterator.next();
|
||||||
|
String[] tmp=key.split("-");
|
||||||
|
String providerId=tmp[0];
|
||||||
|
|
||||||
|
if (providersId.contains(providerId)){
|
||||||
|
Long valueProvider=Long.parseLong(obj.get(key).toString().split(",")[0].replace("[",""));
|
||||||
|
//convert into kb
|
||||||
|
valueProvider=valueProvider/1024;
|
||||||
|
|
||||||
|
Filter filter= new Filter("providerId",providerId );
|
||||||
|
if (!ret.containsKey(filter) ){
|
||||||
|
SortedMap<Calendar, Long> infos = new TreeMap<>();
|
||||||
|
infos.put(calendar, valueProvider);
|
||||||
|
ret.put(filter, infos);
|
||||||
|
}else{
|
||||||
|
SortedMap<Calendar, Long> singleValue=ret.get(filter);
|
||||||
|
if (!singleValue.containsKey(calendar)){
|
||||||
|
//if not exist put
|
||||||
|
singleValue.put(calendar, valueProvider);
|
||||||
|
}else{
|
||||||
|
//if exist, update
|
||||||
|
singleValue.put(calendar, singleValue.get(calendar)+valueProvider);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//logger.trace("return map:{}",map.toString());
|
||||||
|
logger.trace("return ret:{}",ret.toString());
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,7 @@ import org.gcube.accounting.analytics.UsageServiceValue;
|
||||||
import org.gcube.accounting.analytics.UsageStorageValue;
|
import org.gcube.accounting.analytics.UsageStorageValue;
|
||||||
import org.gcube.accounting.analytics.UsageValue;
|
import org.gcube.accounting.analytics.UsageValue;
|
||||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
|
||||||
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryFactory;
|
||||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||||
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
|
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
|
||||||
import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord;
|
import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord;
|
||||||
|
@ -88,6 +89,7 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
//ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
//ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
||||||
//ScopeProvider.instance.set("/gcube/devNext");
|
//ScopeProvider.instance.set("/gcube/devNext");
|
||||||
ScopeProvider.instance.set("/gcube");
|
ScopeProvider.instance.set("/gcube");
|
||||||
|
//ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||||
|
|
||||||
AccountingPersistenceBackendQueryConfiguration configuration = new
|
AccountingPersistenceBackendQueryConfiguration configuration = new
|
||||||
AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchBase.class);
|
AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchBase.class);
|
||||||
|
@ -161,14 +163,14 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
public void testTimeSeriesNoContext() throws Exception{
|
public void testTimeSeriesNoContext() throws Exception{
|
||||||
|
|
||||||
Calendar startTime = Calendar.getInstance();
|
Calendar startTime = Calendar.getInstance();
|
||||||
startTime.set(2015, Calendar.AUGUST, 20);
|
startTime.set(2017, Calendar.APRIL, 20);
|
||||||
Calendar endTime = Calendar.getInstance();
|
Calendar endTime = Calendar.getInstance();
|
||||||
endTime.set(2016, Calendar.SEPTEMBER, 29,23,59);
|
endTime.set(2017, Calendar.APRIL, 29,23,59);
|
||||||
List<Filter> filters = new ArrayList<Filter>();
|
List<Filter> filters = new ArrayList<Filter>();
|
||||||
|
|
||||||
Filter filter =
|
Filter filter =
|
||||||
new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService");
|
new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService");
|
||||||
filters.add(filter);
|
//filters.add(filter);
|
||||||
TemporalConstraint temporalConstraint =
|
TemporalConstraint temporalConstraint =
|
||||||
new TemporalConstraint(startTime.getTimeInMillis(),
|
new TemporalConstraint(startTime.getTimeInMillis(),
|
||||||
endTime.getTimeInMillis(), AggregationMode.DAILY);
|
endTime.getTimeInMillis(), AggregationMode.DAILY);
|
||||||
|
@ -242,7 +244,7 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
@Test
|
@Test
|
||||||
public void getUsageValueQuotaTotal() throws Exception{
|
public void getUsageValueQuotaTotal() throws Exception{
|
||||||
Calendar startTime = Calendar.getInstance();
|
Calendar startTime = Calendar.getInstance();
|
||||||
startTime.set(2015, Calendar.MAY, 1);
|
startTime.set(2017, Calendar.MAY, 1);
|
||||||
Calendar endTime = Calendar.getInstance();
|
Calendar endTime = Calendar.getInstance();
|
||||||
|
|
||||||
|
|
||||||
|
@ -286,7 +288,7 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
Calendar startTimeStorage = Calendar.getInstance();
|
Calendar startTimeStorage = Calendar.getInstance();
|
||||||
startTimeStorage.set(2015, Calendar.MAY, 1);
|
startTimeStorage.set(2015, Calendar.MAY, 1);
|
||||||
Calendar endTimeStorage = Calendar.getInstance();
|
Calendar endTimeStorage = Calendar.getInstance();
|
||||||
endTimeStorage.set(2017, Calendar.FEBRUARY, 21);
|
endTimeStorage.set(2017, Calendar.APRIL, 13);
|
||||||
/*TemporalConstraint temporalConstraintStorage =new TemporalConstraint(startTimeStorage.getTimeInMillis(),
|
/*TemporalConstraint temporalConstraintStorage =new TemporalConstraint(startTimeStorage.getTimeInMillis(),
|
||||||
endTimeStorage.getTimeInMillis(), AggregationMode.DAILY);*/
|
endTimeStorage.getTimeInMillis(), AggregationMode.DAILY);*/
|
||||||
TemporalConstraint temporalConstraintStorage =null;
|
TemporalConstraint temporalConstraintStorage =null;
|
||||||
|
@ -297,11 +299,11 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
|
|
||||||
//richiedo la lista di dati usati totali
|
//richiedo la lista di dati usati totali
|
||||||
List<UsageValue> listTotalFilter=new ArrayList<UsageValue>();
|
List<UsageValue> listTotalFilter=new ArrayList<UsageValue>();
|
||||||
/*listTotalFilter.add(totalfilter);
|
listTotalFilter.add(totalfilter);
|
||||||
listTotalFilter.add(totalfilterStorageStatus);
|
listTotalFilter.add(totalfilterStorageStatus);
|
||||||
listTotalFilter.add(totalfilterStorageStatus_2);
|
listTotalFilter.add(totalfilterStorageStatus_2);
|
||||||
listTotalFilter.add(totalfilterStorageStatus_3);
|
listTotalFilter.add(totalfilterStorageStatus_3);
|
||||||
listTotalFilter.add(totalfilterStorageStatus_4);*/
|
listTotalFilter.add(totalfilterStorageStatus_4);
|
||||||
listTotalFilter.add(totalfilterStorageStatusPeriod);
|
listTotalFilter.add(totalfilterStorageStatusPeriod);
|
||||||
|
|
||||||
|
|
||||||
|
@ -335,7 +337,6 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
Calendar endTime = Calendar.getInstance();
|
Calendar endTime = Calendar.getInstance();
|
||||||
endTime.set(2016, Calendar.SEPTEMBER, 28,23,59);
|
endTime.set(2016, Calendar.SEPTEMBER, 28,23,59);
|
||||||
|
|
||||||
|
|
||||||
List<Filter> filters = new ArrayList<Filter>();
|
List<Filter> filters = new ArrayList<Filter>();
|
||||||
//filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_NAME, "IS-Registry"));
|
//filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_NAME, "IS-Registry"));
|
||||||
//filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_CLASS, "Common"));
|
//filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_CLASS, "Common"));
|
||||||
|
@ -498,11 +499,11 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
@Test
|
@Test
|
||||||
public void testContextService() throws Exception {
|
public void testContextService() throws Exception {
|
||||||
Calendar startTime = Calendar.getInstance();
|
Calendar startTime = Calendar.getInstance();
|
||||||
startTime.set(2016, Calendar.SEPTEMBER, 1);
|
startTime.set(2017, Calendar.APRIL, 20);
|
||||||
Calendar endTime = Calendar.getInstance();
|
Calendar endTime = Calendar.getInstance();
|
||||||
endTime.set(2016, Calendar.OCTOBER, 20,23,59);
|
endTime.set(2017, Calendar.APRIL, 28,23,59);
|
||||||
List<Filter> filters = new ArrayList<Filter>();
|
List<Filter> filters = new ArrayList<Filter>();
|
||||||
filters.add(new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService"));
|
//filters.add(new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService"));
|
||||||
//filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
|
//filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
|
||||||
TemporalConstraint temporalConstraint =
|
TemporalConstraint temporalConstraint =
|
||||||
new TemporalConstraint(startTime.getTimeInMillis(),
|
new TemporalConstraint(startTime.getTimeInMillis(),
|
||||||
|
@ -517,6 +518,7 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
// , /gcube/devsec/TestCreation12, /gcube/devsec/TestCreation13, /gcube/devsec/TestCreation14, /gcube/devsec/TestCreation15, /gcube/devsec/TestCreation16, /gcube/devsec/TestCreation17, /gcube/devsec/TestCreation7, /gcube/devsec/TestCreation8, /gcube/devsec/TestCreation9, /gcube/devsec/TestLucio2]]
|
// , /gcube/devsec/TestCreation12, /gcube/devsec/TestCreation13, /gcube/devsec/TestCreation14, /gcube/devsec/TestCreation15, /gcube/devsec/TestCreation16, /gcube/devsec/TestCreation17, /gcube/devsec/TestCreation7, /gcube/devsec/TestCreation8, /gcube/devsec/TestCreation9, /gcube/devsec/TestLucio2]]
|
||||||
context.add("/gcube");
|
context.add("/gcube");
|
||||||
context.add("/gcube/devNext");
|
context.add("/gcube/devNext");
|
||||||
|
/*
|
||||||
context.add("/gcube/devNext/Luciotest");
|
context.add("/gcube/devNext/Luciotest");
|
||||||
context.add("/gcube/devNext/NextNext");
|
context.add("/gcube/devNext/NextNext");
|
||||||
context.add("/gcube/devsec");
|
context.add("/gcube/devsec");
|
||||||
|
@ -528,6 +530,7 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
context.add("/gcube/devsec/preVRE");
|
context.add("/gcube/devsec/preVRE");
|
||||||
context.add("/gcube/preprod/preVRE");
|
context.add("/gcube/preprod/preVRE");
|
||||||
context.add("/gcube/preprod");
|
context.add("/gcube/preprod");
|
||||||
|
*/
|
||||||
SortedMap<Filter, SortedMap<Calendar, Info>> setContext =
|
SortedMap<Filter, SortedMap<Calendar, Info>> setContext =
|
||||||
accountingPersistenceQueryCouchBase.getContextTimeSeries(
|
accountingPersistenceQueryCouchBase.getContextTimeSeries(
|
||||||
clz, temporalConstraint, filters,
|
clz, temporalConstraint, filters,
|
||||||
|
@ -554,8 +557,139 @@ public class AccountingPersistenceQueryCouchBaseTest {
|
||||||
parameters.add("serviceName");
|
parameters.add("serviceName");
|
||||||
Class<AggregatedServiceUsageRecord> clz =
|
Class<AggregatedServiceUsageRecord> clz =
|
||||||
AggregatedServiceUsageRecord.class;
|
AggregatedServiceUsageRecord.class;
|
||||||
SortedMap<String,Integer> result= accountingPersistenceQueryCouchBase.getListUsage(clz,temporalConstraint,
|
// SortedMap<String,Integer> result= accountingPersistenceQueryCouchBase.getListUsage(clz,temporalConstraint, filters,context,parameters);
|
||||||
filters,context,parameters);
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void getRecord() throws Exception{
|
||||||
|
String recordId="91e1c339-d811-45d7-a13f-7385af59e3c8";
|
||||||
|
String type="service";
|
||||||
|
|
||||||
|
String document=accountingPersistenceQueryCouchBase.getRecord(recordId, type);
|
||||||
|
logger.debug("document:{}",document);
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void getQuerableKeyStorageStatus() throws Exception{
|
||||||
|
SortedSet<String> keys;
|
||||||
|
keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedStorageStatusRecord.class);
|
||||||
|
for (String key : keys) {
|
||||||
|
if (key != null && !key.isEmpty()) {
|
||||||
|
logger.debug("key:" +key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.debug("List FilterKeys:" + keys.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetSpaceProvidersIds(){
|
||||||
|
SortedSet<String> listProvidersId =
|
||||||
|
accountingPersistenceQueryCouchBase.getSpaceProvidersIds();
|
||||||
|
logger.debug("Result final{}", listProvidersId);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetFilterValue() throws Exception{
|
||||||
|
|
||||||
|
//String key="consumerId";
|
||||||
|
String key="dataServiceId";
|
||||||
|
|
||||||
|
SortedSet<NumberedFilter> listFilterValue =
|
||||||
|
accountingPersistenceQueryCouchBase.getFilterValues(AggregatedStorageStatusRecord.class, null, null, key);
|
||||||
|
logger.debug("Result final{}", listFilterValue);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUsageStorage() throws Exception {
|
||||||
|
Calendar startTime = Calendar.getInstance();
|
||||||
|
startTime.set(2017, Calendar.APRIL, 13);
|
||||||
|
Calendar endTime = Calendar.getInstance();
|
||||||
|
endTime.set(2017, Calendar.APRIL, 26,23,59);
|
||||||
|
List<Filter> filters = new ArrayList<Filter>();
|
||||||
|
//filters.add(new Filter(AggregatedStorageStatusRecord.DATA_SERVICEID, "identifier"));
|
||||||
|
filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
|
||||||
|
TemporalConstraint temporalConstraint =
|
||||||
|
new TemporalConstraint(startTime.getTimeInMillis(),
|
||||||
|
endTime.getTimeInMillis(), AggregationMode.DAILY);
|
||||||
|
|
||||||
|
|
||||||
|
List<String> providerId=new ArrayList<String>();
|
||||||
|
Class<AggregatedStorageStatusRecord> clz =
|
||||||
|
AggregatedStorageStatusRecord.class;
|
||||||
|
|
||||||
|
//providerId.add("Rstudio");
|
||||||
|
providerId.add("MongoDb");
|
||||||
|
|
||||||
|
SortedMap<Filter, SortedMap<Calendar, Long>> setContext =
|
||||||
|
accountingPersistenceQueryCouchBase.getSpaceTimeSeries(
|
||||||
|
clz,
|
||||||
|
temporalConstraint, filters,
|
||||||
|
providerId);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
int count = setContext.size();
|
||||||
|
Filter firstRemovalKey = null;
|
||||||
|
for(Filter nf : setContext.keySet()){
|
||||||
|
if(--count>=0){
|
||||||
|
//if(pad){
|
||||||
|
padMapStorage(setContext.get(nf), temporalConstraint);
|
||||||
|
|
||||||
|
//}
|
||||||
|
}else{
|
||||||
|
if(firstRemovalKey==null){
|
||||||
|
firstRemovalKey = nf;
|
||||||
|
}else{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
if(firstRemovalKey!=null){
|
||||||
|
return setContext.subMap(setContext.firstKey(), firstRemovalKey);
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
logger.debug("Result final{}", setContext);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public SortedMap<Calendar, Long> padMapStorage(
|
||||||
|
SortedMap<Calendar, Long> unpaddedData,
|
||||||
|
TemporalConstraint temporalConstraint) throws Exception {
|
||||||
|
|
||||||
|
//JSONObject jsonObject = getPaddingJSONObject(unpaddedData);
|
||||||
|
SortedSet<Calendar> sequence = temporalConstraint.getCalendarSequence();
|
||||||
|
Long longValuePre = null;
|
||||||
|
for (Calendar progressTime : sequence) {
|
||||||
|
Long longValue = unpaddedData.get(progressTime);
|
||||||
|
|
||||||
|
if (longValue == null) {
|
||||||
|
unpaddedData.put(progressTime, longValuePre);
|
||||||
|
}else{
|
||||||
|
longValuePre=longValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
return unpaddedData;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Reference in New Issue