|
|
|
|
@ -5,28 +5,20 @@ package org.gcube.accounting.analytics.persistence.couchdb;
|
|
|
|
|
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Calendar;
|
|
|
|
|
import java.util.HashMap;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
import java.util.SortedMap;
|
|
|
|
|
import java.util.SortedSet;
|
|
|
|
|
|
|
|
|
|
import org.codehaus.jackson.JsonNode;
|
|
|
|
|
import org.codehaus.jackson.map.ObjectMapper;
|
|
|
|
|
import org.codehaus.jackson.node.ArrayNode;
|
|
|
|
|
import org.ektorp.DocumentNotFoundException;
|
|
|
|
|
import org.ektorp.ViewQuery;
|
|
|
|
|
import org.ektorp.ViewResult;
|
|
|
|
|
import org.gcube.accounting.analytics.Filter;
|
|
|
|
|
import org.gcube.accounting.analytics.Info;
|
|
|
|
|
import org.gcube.accounting.analytics.NumberedFilter;
|
|
|
|
|
import org.gcube.accounting.analytics.TemporalConstraint;
|
|
|
|
|
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
|
|
|
|
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
|
|
|
|
|
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
|
|
|
|
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
|
|
|
|
|
import org.gcube.common.scope.api.ScopeProvider;
|
|
|
|
|
import org.gcube.documentstore.records.AggregatedRecord;
|
|
|
|
|
import org.gcube.testutility.ScopedTest;
|
|
|
|
|
import org.json.JSONArray;
|
|
|
|
|
import org.json.JSONObject;
|
|
|
|
|
import org.junit.Before;
|
|
|
|
|
import org.junit.Test;
|
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
|
@ -39,39 +31,8 @@ import org.slf4j.LoggerFactory;
|
|
|
|
|
public class AccountingPersistenceQueryCouchDBTest extends ScopedTest {
|
|
|
|
|
|
|
|
|
|
private static Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryCouchDBTest.class);
|
|
|
|
|
|
|
|
|
|
protected AccountingPersistenceQueryCouchDB accountingPersistenceQueryCouchDB;
|
|
|
|
|
|
|
|
|
|
public class ExtendedInfo extends Info {
|
|
|
|
|
|
|
|
|
|
protected String key;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @return the key
|
|
|
|
|
*/
|
|
|
|
|
public String getKey() {
|
|
|
|
|
return key;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @param key the key to set
|
|
|
|
|
*/
|
|
|
|
|
public void setKey(String key) {
|
|
|
|
|
this.key = key;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public ExtendedInfo(String key, Calendar calendar, JSONObject value){
|
|
|
|
|
super(calendar, value);
|
|
|
|
|
this.key = key;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public String toString(){
|
|
|
|
|
String info = String .format("Key : %s, %s ", key, super.toString());
|
|
|
|
|
return info;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected AccountingPersistenceQueryCouchDB accountingPersistenceQueryCouchDBNew;
|
|
|
|
|
|
|
|
|
|
@Before
|
|
|
|
|
public void before() throws Exception{
|
|
|
|
|
@ -79,144 +40,79 @@ public class AccountingPersistenceQueryCouchDBTest extends ScopedTest {
|
|
|
|
|
ScopeProvider.instance.set("/gcube");
|
|
|
|
|
AccountingPersistenceBackendQueryConfiguration configuration = new
|
|
|
|
|
AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchDB.class);
|
|
|
|
|
configuration.addProperty(AccountingPersistenceQueryCouchDB.DB_NAME, "accounting-my-dev-test");
|
|
|
|
|
accountingPersistenceQueryCouchDB = new AccountingPersistenceQueryCouchDB();
|
|
|
|
|
accountingPersistenceQueryCouchDB.prepareConnection(configuration);
|
|
|
|
|
|
|
|
|
|
configuration.addProperty(AccountingPersistenceQueryCouchDB.DB_NAME, "my_couchapp_test");
|
|
|
|
|
String url = configuration.getProperty(AccountingPersistenceQueryCouchDB.URL_PROPERTY_KEY);
|
|
|
|
|
url = url.replace("accounting-d-d4s.d4science.org", "couchdb01-d-d4s.d4science.org:5984");
|
|
|
|
|
configuration.addProperty(AccountingPersistenceQueryCouchDB.URL_PROPERTY_KEY, url);
|
|
|
|
|
accountingPersistenceQueryCouchDBNew = new AccountingPersistenceQueryCouchDB();
|
|
|
|
|
accountingPersistenceQueryCouchDBNew.prepareConnection(configuration);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected Map<Calendar, Info> myNewQuery(
|
|
|
|
|
@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass,
|
|
|
|
|
TemporalConstraint temporalConstraint, List<Filter> filters)
|
|
|
|
|
throws Exception {
|
|
|
|
|
|
|
|
|
|
//String currentScope = BasicUsageRecord.getScopeFromToken();
|
|
|
|
|
String currentScope = ScopeProvider.instance.get();
|
|
|
|
|
String designDocId = accountingPersistenceQueryCouchDB.getDesignDocId(recordClass) + "TopValues";
|
|
|
|
|
|
|
|
|
|
ViewQuery query = new ViewQuery().designDocId(designDocId);
|
|
|
|
|
|
|
|
|
|
query.group(true);
|
|
|
|
|
int groupLevel = temporalConstraint.getAggregationMode().ordinal()+2;
|
|
|
|
|
if(filters!=null){
|
|
|
|
|
groupLevel += filters.size()*2;
|
|
|
|
|
}
|
|
|
|
|
query.groupLevel(groupLevel);
|
|
|
|
|
|
|
|
|
|
ArrayNode startKey = new ObjectMapper().createArrayNode();
|
|
|
|
|
startKey.add(currentScope);
|
|
|
|
|
ArrayNode endKey = new ObjectMapper().createArrayNode();
|
|
|
|
|
endKey.add(currentScope);
|
|
|
|
|
|
|
|
|
|
String viewName = AccountingPersistenceQueryCouchDB.MAP_REDUCE_ALL;
|
|
|
|
|
query = query.viewName(viewName);
|
|
|
|
|
|
|
|
|
|
startKey = accountingPersistenceQueryCouchDB.getRangeKey(startKey, temporalConstraint.getStartTime(), temporalConstraint.getAggregationMode(), false);
|
|
|
|
|
endKey = accountingPersistenceQueryCouchDB.getRangeKey(endKey, temporalConstraint.getEndTime(), temporalConstraint.getAggregationMode(), false);
|
|
|
|
|
|
|
|
|
|
if(filters!=null && filters.size()!=0){
|
|
|
|
|
for(Filter filter : filters){
|
|
|
|
|
/*
|
|
|
|
|
ObjectNode objectNode = new ObjectMapper().createObjectNode();
|
|
|
|
|
objectNode.put(filter.getKey(), filter.getValue());
|
|
|
|
|
startKey.add(objectNode);
|
|
|
|
|
endKey.add(objectNode);
|
|
|
|
|
*/
|
|
|
|
|
startKey.add(filter.getKey());
|
|
|
|
|
endKey.add(filter.getKey());
|
|
|
|
|
|
|
|
|
|
if(filter.getValue()!=null && filters.size()==1){
|
|
|
|
|
startKey.add(filter.getValue());
|
|
|
|
|
endKey.add(filter.getValue());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
//endKey.add("{}");
|
|
|
|
|
|
|
|
|
|
query.startKey(startKey);
|
|
|
|
|
query.endKey(endKey);
|
|
|
|
|
query.descending(true);
|
|
|
|
|
|
|
|
|
|
logger.trace("\nDesign Doc ID : {}, View Name : {}, Group Level : {} \nStart Key : {} \nEnd Key : {}",
|
|
|
|
|
designDocId, viewName, groupLevel, startKey, endKey);
|
|
|
|
|
|
|
|
|
|
Map<Calendar, Info> infos = new HashMap<Calendar, Info>();
|
|
|
|
|
|
|
|
|
|
ViewResult viewResult;
|
|
|
|
|
try {
|
|
|
|
|
viewResult = accountingPersistenceQueryCouchDB.query(query);
|
|
|
|
|
} catch (DocumentNotFoundException e) {
|
|
|
|
|
// Install VIEW if valid and does not exist
|
|
|
|
|
throw e;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (ViewResult.Row row : viewResult) {
|
|
|
|
|
JsonNode key = row.getKeyAsNode();
|
|
|
|
|
JSONArray array = new JSONArray(key.toString());
|
|
|
|
|
Calendar calendar = accountingPersistenceQueryCouchDB.getCalendarFromArray(array);
|
|
|
|
|
|
|
|
|
|
JsonNode value = row.getValueAsNode();
|
|
|
|
|
JSONObject obj = new JSONObject(value.toString());
|
|
|
|
|
|
|
|
|
|
ExtendedInfo extendedInfo = new ExtendedInfo(key.toString(), calendar, obj);
|
|
|
|
|
|
|
|
|
|
infos.put(calendar, extendedInfo);
|
|
|
|
|
public static <V> String printCalendarMap(Map<Calendar, V> map){
|
|
|
|
|
StringBuilder stringBuilder = new StringBuilder();
|
|
|
|
|
for(Calendar calendar : map.keySet()){
|
|
|
|
|
V v = map.get(calendar);
|
|
|
|
|
stringBuilder.append(v.toString());
|
|
|
|
|
stringBuilder.append("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return infos;
|
|
|
|
|
return stringBuilder.toString();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void printMap(Map<Calendar, Info> map){
|
|
|
|
|
for(Info info : map.values()){
|
|
|
|
|
logger.debug("{}", info);
|
|
|
|
|
public static <T, V> String printMap(Map<T, ? extends Map<Calendar, V>> map){
|
|
|
|
|
StringBuilder stringBuilder = new StringBuilder();
|
|
|
|
|
for(T t : map.keySet()) {
|
|
|
|
|
stringBuilder.append(t.toString());
|
|
|
|
|
stringBuilder.append("\n");
|
|
|
|
|
Map<Calendar, V> innerMap = map.get(t);
|
|
|
|
|
stringBuilder.append(printCalendarMap(innerMap));
|
|
|
|
|
stringBuilder.append("\n");
|
|
|
|
|
}
|
|
|
|
|
return stringBuilder.toString();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
public void test() throws Exception {
|
|
|
|
|
public void testTop() throws Exception {
|
|
|
|
|
Calendar startTime = Calendar.getInstance();
|
|
|
|
|
startTime.set(2015, Calendar.OCTOBER, 9);
|
|
|
|
|
Calendar endTime = Calendar.getInstance();
|
|
|
|
|
endTime.set(2015, Calendar.OCTOBER, 16);
|
|
|
|
|
|
|
|
|
|
List<Filter> filters = new ArrayList<Filter>();
|
|
|
|
|
Filter filter = new Filter("consumerId", "gianpaolo.coro");
|
|
|
|
|
Filter filter = new Filter("consumerId", null);
|
|
|
|
|
filters.add(filter);
|
|
|
|
|
TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
|
|
|
|
|
Map<Calendar, Info> oldQuery = accountingPersistenceQueryCouchDB.reallyQuery(AggregatedServiceUsageRecord.class, temporalConstraint, filters);
|
|
|
|
|
logger.debug("Old Query Results :");
|
|
|
|
|
printMap(oldQuery);
|
|
|
|
|
|
|
|
|
|
TemporalConstraint temporalConstraint =
|
|
|
|
|
new TemporalConstraint(startTime.getTimeInMillis(),
|
|
|
|
|
endTime.getTimeInMillis(), AggregationMode.DAILY);
|
|
|
|
|
|
|
|
|
|
Map<Calendar, Info> newQuery = myNewQuery(AggregatedServiceUsageRecord.class, temporalConstraint, filters);
|
|
|
|
|
logger.debug("New Query Results :");
|
|
|
|
|
printMap(newQuery);
|
|
|
|
|
logger.trace("\n\n");
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
public void getPossibleValuesForKeyTest() throws Exception {
|
|
|
|
|
int limit = 100;
|
|
|
|
|
Class<AggregatedServiceUsageRecord> clz =
|
|
|
|
|
AggregatedServiceUsageRecord.class;
|
|
|
|
|
|
|
|
|
|
Map<String, Integer> values = accountingPersistenceQueryCouchDB.getValuesAndOccurrences(AggregatedServiceUsageRecord.class, AggregatedServiceUsageRecord.CONSUMER_ID);
|
|
|
|
|
logger.debug("Values and Occurrences for {} for key {} : {}",
|
|
|
|
|
AggregatedServiceUsageRecord.class.getSimpleName(),
|
|
|
|
|
AggregatedServiceUsageRecord.CONSUMER_ID, values);
|
|
|
|
|
Map<NumberedFilter, SortedMap<Calendar, Info>> map =
|
|
|
|
|
accountingPersistenceQueryCouchDBNew.getTopValues(clz,
|
|
|
|
|
temporalConstraint, filters);
|
|
|
|
|
|
|
|
|
|
values = accountingPersistenceQueryCouchDB.getValuesAndOccurrences(AggregatedServiceUsageRecord.class, AggregatedServiceUsageRecord.CONSUMER_ID, limit);
|
|
|
|
|
logger.debug("Values and Occurrences (limit {}) for {} for key {} : {}",
|
|
|
|
|
limit, AggregatedServiceUsageRecord.class.getSimpleName(),
|
|
|
|
|
AggregatedServiceUsageRecord.CONSUMER_ID, values);
|
|
|
|
|
logger.debug("Top Query Results :\n{}", printMap(map));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
values = accountingPersistenceQueryCouchDB.getValuesAndOccurrences(AggregatedStorageUsageRecord.class, AggregatedStorageUsageRecord.CONSUMER_ID);
|
|
|
|
|
logger.debug("Values and Occurrences for {} for key {} : {}",
|
|
|
|
|
AggregatedStorageUsageRecord.class.getSimpleName(),
|
|
|
|
|
AggregatedStorageUsageRecord.CONSUMER_ID, values);
|
|
|
|
|
SortedSet<NumberedFilter> possibleValues =
|
|
|
|
|
accountingPersistenceQueryCouchDBNew.getNextPossibleValues(clz,
|
|
|
|
|
temporalConstraint, filters);
|
|
|
|
|
|
|
|
|
|
logger.debug("Possibile Values :\n{}", possibleValues);
|
|
|
|
|
|
|
|
|
|
filter.setValue("UNKNOWN");
|
|
|
|
|
SortedMap<Calendar, Info> timeSeries =
|
|
|
|
|
accountingPersistenceQueryCouchDBNew.getTimeSeries(clz,
|
|
|
|
|
temporalConstraint, filters);
|
|
|
|
|
|
|
|
|
|
logger.debug("Time Series for {}:\n{}", filter,
|
|
|
|
|
printCalendarMap(timeSeries));
|
|
|
|
|
|
|
|
|
|
values = accountingPersistenceQueryCouchDB.getValuesAndOccurrences(AggregatedStorageUsageRecord.class, AggregatedStorageUsageRecord.CONSUMER_ID, limit);
|
|
|
|
|
logger.debug("Values and Occurrences (limit {}) for {} for key {} : {}",
|
|
|
|
|
limit, AggregatedStorageUsageRecord.class.getSimpleName(),
|
|
|
|
|
AggregatedStorageUsageRecord.CONSUMER_ID, values);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
