Alessandro Pieve 7 years ago
parent e8bf0e29be
commit 4268109e10

@ -65,7 +65,6 @@ import com.couchbase.client.java.query.dsl.Expression;
import com.couchbase.client.java.query.dsl.Sort;
import com.couchbase.client.java.query.dsl.path.GroupByPath;
import com.couchbase.client.java.query.dsl.path.OffsetPath;
import com.couchbase.client.java.view.DesignDocument;
import com.couchbase.client.java.view.ViewQuery;
import com.couchbase.client.java.view.ViewResult;
import com.couchbase.client.java.view.ViewRow;
@ -87,8 +86,8 @@ AccountingPersistenceBackendQuery {
public static final String BUCKET_STORAGE_NAME_PROPERTY_KEY="AggregatedStorageUsageRecord";
public static final String BUCKET_STORAGESTATUS_NAME_PROPERTY_KEY="AggregatedStorageStatusRecord";
public static final String BUCKET_SERVICE_NAME_PROPERTY_KEY="AggregatedServiceUsageRecord";
public static final String BUCKET_PORTLET_NAME_PROPERTY_KEY="AggregatedPortletUsageRecord";
@ -112,10 +111,10 @@ AccountingPersistenceBackendQuery {
protected Bucket bucketStorage;
protected String bucketNameStorage;
protected Bucket bucketStorageStatus;
protected String bucketNameStorageStatus;
protected Bucket bucketService;
protected String bucketNameService;
@ -153,7 +152,7 @@ AccountingPersistenceBackendQuery {
bucketNameStorage = configuration.getProperty(BUCKET_STORAGE_NAME_PROPERTY_KEY);
bucketNameStorageStatus = configuration.getProperty(BUCKET_STORAGESTATUS_NAME_PROPERTY_KEY);
bucketNameService = configuration.getProperty(BUCKET_SERVICE_NAME_PROPERTY_KEY);
bucketNameJob = configuration.getProperty(BUCKET_JOB_NAME_PROPERTY_KEY);
bucketNamePortlet = configuration.getProperty(BUCKET_PORTLET_NAME_PROPERTY_KEY);
@ -168,7 +167,7 @@ AccountingPersistenceBackendQuery {
connectionMap.put(BUCKET_STORAGESTATUS_NAME_PROPERTY_KEY, bucketStorageStatus);
logger.debug("connectionMap"+connectionMap.toString());
bucketService = cluster.openBucket(bucketNameService, password);
connectionMap.put(BUCKET_SERVICE_NAME_PROPERTY_KEY, bucketService);
@ -414,12 +413,12 @@ AccountingPersistenceBackendQuery {
reduceFunction = null;
String property = collection.iterator().next();
reduceFunction = property;
}
return reduceFunction;
}
/*
* EXPERIMENTAL
*
@ -567,7 +566,7 @@ AccountingPersistenceBackendQuery {
else
currentScope = context;
JsonArray startKey = JsonArray.create();
JsonArray endKey = JsonArray.create();
//no scope call a map reduce without scope in startkey and endkey
if (!noScope){
@ -701,7 +700,7 @@ AccountingPersistenceBackendQuery {
logger.warn("not execute query",e.getLocalizedMessage());
//throw e;
}
try {
if(viewResult.totalRows()!=0){
ViewRow row=viewResult.allRows().get(0);
@ -709,20 +708,20 @@ AccountingPersistenceBackendQuery {
temporalConstraint.getStartTime(),
aggregationMode, false, false);
Calendar calendar = getCalendarFromArray(array);
JsonObject value = (JsonObject) row.value();
JSONObject objJson= new JSONObject(value.toString());
JSONObject objJsontemplate=new JSONObject();
Iterator<?> iterateJson = objJson.keys();
while( iterateJson.hasNext() ) {
String key = (String)iterateJson.next();
objJsontemplate.put(key, 0);
}
Iterator<?> iterateJson = objJson.keys();
while( iterateJson.hasNext() ) {
String key = (String)iterateJson.next();
objJsontemplate.put(key, 0);
}
//generate an example object for json
Info info = new Info(calendar, objJsontemplate);
infos.put(calendar, info);
}
} catch (Exception e) {
logger.warn("error :{}",e.getLocalizedMessage());
}
@ -746,7 +745,7 @@ AccountingPersistenceBackendQuery {
return map;
}
@Override
public SortedMap<Calendar, Info> getNoContextTimeSeries(
Class<? extends AggregatedRecord<?, ?>> clz,
@ -789,10 +788,10 @@ AccountingPersistenceBackendQuery {
SortedSet<NumberedFilter> top=null;
//top = getNextPossibleValues(clz,temporalConstraint, filters, topKey, orderingProperty);
if (usingNextPossibleValuesWithMap(clz,topKey,filters)){
logger.trace("getNextPossibleValues using map");
top = getNextPossibleValuesWithMap(clz,temporalConstraint, filters, topKey, orderingProperty);
@ -808,7 +807,7 @@ AccountingPersistenceBackendQuery {
ret.put(nf, map);
filters.remove(nf);
}
return ret;
}
@ -867,8 +866,8 @@ AccountingPersistenceBackendQuery {
logger.debug("usingNextPossibleValuesWithMap complete key and name");
String viewName=getMapReduceFunctionName(keys);
String designDocId =DESIGN_DOC_ID+getDesignDocIdName(keys);
BucketManager bucketManager = connectionMap.get(clz.getSimpleName()).bucketManager();
//logger.debug("----"+bucketManager.getDesignDocument(designDocId));
if (bucketManager.getDesignDocument(designDocId)!=null){
@ -889,7 +888,7 @@ AccountingPersistenceBackendQuery {
} catch (Exception e) {
return false;
}
*/
*/
return true;
}
@ -964,11 +963,11 @@ AccountingPersistenceBackendQuery {
Sort sort = Sort.desc(orderingProperty);
OffsetPath path = select(selectExpressionArray).from(connectionMap.get(clz.getSimpleName()).name())
.where(whereExpression).groupBy(key).orderBy(sort);
/*END DELETE ONLY FOR TEST*/
if (filters != null && filters.size() != 0) {
// Sorting filter for call a mapreduce
@ -1076,7 +1075,7 @@ AccountingPersistenceBackendQuery {
ret.add(numberedFilter);
}
return ret;
}
@ -1170,9 +1169,9 @@ AccountingPersistenceBackendQuery {
.where(whereExpression).groupBy(key).orderBy(sort);
logger.debug("Query for top:"+path.toString());
Comparator<NumberedFilter> comparator = new Comparator<NumberedFilter>() {
@Override
public int compare(NumberedFilter o1, NumberedFilter o2) {
@ -1250,7 +1249,7 @@ AccountingPersistenceBackendQuery {
if (!map.isEmpty()){
ret.put(nf, map);
}
filters.remove(nf);
}
return ret;
@ -1475,13 +1474,13 @@ AccountingPersistenceBackendQuery {
for (UsageValue totalFilters:listUsage){
logger.debug("-----------------/ init for totalFilters");
/*
UsageValue totalfilter=
new UsageServiceValue(context,"lucio.lelii",AggregatedServiceUsageRecord.class,temporalConstraint,filters);
*/
*/
String currentScope = totalFilters.getContext();
@ -1494,6 +1493,7 @@ AccountingPersistenceBackendQuery {
logger.trace("temporalConstraint:{}",totalFilters);
TemporalConstraint temporalConstraint=totalFilters.getTemporalConstraint();
if (temporalConstraint==(null)){
//used for no temporalConstraint
logger.trace("Not found temporalConstraint");
@ -1509,14 +1509,15 @@ AccountingPersistenceBackendQuery {
if (totalFilters.getClz().getSimpleName().equals(AggregatedStorageStatusRecord.class.getSimpleName())){
//TODO sarebbe possibile invece con il nome verificare che getClz e' un istanza di AggregatedStorageStatusRecord?
designDocId = "QuotaTotal";
}
}
else if (totalFilters.getClz().getSimpleName().equals(AggregatedStorageStatusRecord.class.getSimpleName())){
logger.trace("AggregatedStorageStatusRecord with temporalConstraint");
designDocId = "Quota";
}
AggregationMode aggregationMode = temporalConstraint.getAggregationMode();
temporalStartKey = getRangeKey(
@ -1535,124 +1536,137 @@ AccountingPersistenceBackendQuery {
//countFilters =totalFiltersService.getFiltersValue().size();
}
//do {
String viewNameTmp=null;
JsonArray startKeyTmp=JsonArray.create();
startKeyTmp.add(currentScope);
JsonArray endKeyTmp=JsonArray.create();
endKeyTmp.add(currentScope);
int groupLevelTmp= 2;
//FiltersValue singleFilter=null;
viewNameTmp=AggregatedServiceUsageRecord.CONSUMER_ID;
startKeyTmp.add(totalFilters.getIdentifier());
endKeyTmp.add(totalFilters.getIdentifier());
//FiltersValue singleFilter=null;
if (totalFilters instanceof UsageServiceValue){
//logger.debug("******UsageServiceValue");
UsageServiceValue totalFiltersService = (UsageServiceValue)totalFilters;
//singleFilter=totalFiltersService.getFiltersValue().get(i);
for (Filter filter:totalFiltersService.getFilters()){
viewNameTmp=viewNameTmp+"__"+filter.getKey();
startKeyTmp.add(filter.getValue());
endKeyTmp.add(filter.getValue());
groupLevelTmp++;
}
}
//not defined temporal constraint
for (Object temporal: temporalStartKey.toList()){
if (!temporal.toString().isEmpty())
startKeyTmp.add(temporal);
String viewNameTmp=null;
JsonArray startKeyTmp=JsonArray.create();
startKeyTmp.add(currentScope);
JsonArray endKeyTmp=JsonArray.create();
endKeyTmp.add(currentScope);
int groupLevelTmp= 2;
//FiltersValue singleFilter=null;
viewNameTmp=AggregatedServiceUsageRecord.CONSUMER_ID;
startKeyTmp.add(totalFilters.getIdentifier());
endKeyTmp.add(totalFilters.getIdentifier());
//FiltersValue singleFilter=null;
if (totalFilters instanceof UsageServiceValue){
//logger.debug("******UsageServiceValue");
UsageServiceValue totalFiltersService = (UsageServiceValue)totalFilters;
//singleFilter=totalFiltersService.getFiltersValue().get(i);
for (Filter filter:totalFiltersService.getFilters()){
viewNameTmp=viewNameTmp+"__"+filter.getKey();
startKeyTmp.add(filter.getValue());
endKeyTmp.add(filter.getValue());
groupLevelTmp++;
}
int count =1;
for (Object temporal: temporalEndKey.toList()){
if (!temporal.toString().isEmpty()){
//couchbase excludes last value
if (count==temporalEndKey.size())
temporal=(int)temporal+1;
endKeyTmp.add(temporal);
}
count++;
}
//not defined temporal constraint
for (Object temporal: temporalStartKey.toList()){
if (!temporal.toString().isEmpty())
startKeyTmp.add(temporal);
}
int count =1;
for (Object temporal: temporalEndKey.toList()){
if (!temporal.toString().isEmpty()){
//couchbase excludes last value
if (count==temporalEndKey.size())
temporal=(int)temporal+1;
endKeyTmp.add(temporal);
}
logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, "
+ "Group Level : {}, Start Key : {}, End Key : {},"
+ "temporalStartKey :{}, temporalEndKey :{}",
totalFilters.getClz().getSimpleName(),designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp,temporalStartKey.toString(), temporalEndKey.toString());
logger.trace("connectionMap:{}",connectionMap.toString());
ViewQuery query = ViewQuery.from(designDocId, viewNameTmp);
query.inclusiveEnd();
count++;
}
logger.trace("Bucket :{}, Design Doc ID : {}, View Name : {}, "
+ "Group Level : {}, Start Key : {}, End Key : {},"
+ "temporalStartKey :{}, temporalEndKey :{}",
totalFilters.getClz().getSimpleName(),designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp,temporalStartKey.toString(), temporalEndKey.toString());
logger.trace("connectionMap:{}",connectionMap.toString());
ViewQuery query = ViewQuery.from(designDocId, viewNameTmp);
//if storage
query.inclusiveEnd();
if (designDocId == "QuotaTotal"){
query.startKey(endKeyTmp);
query.endKey(startKeyTmp);
query.descending(true);
query.limit(1);
}
else{
query.groupLevel(groupLevelTmp);
query.startKey(startKeyTmp);
query.endKey(endKeyTmp);
query.descending(false);
}
ViewResult viewResult;
try {
viewResult = connectionMap.get(totalFilters.getClz().getSimpleName()).query(query);
} catch (Exception e) {
logger.error(e.getLocalizedMessage());
throw e;
}
logger.trace("query row:{}",query.toString());
ViewResult viewResult;
try {
viewResult = connectionMap.get(totalFilters.getClz().getSimpleName()).query(query);
} catch (Exception e) {
logger.error(e.getLocalizedMessage());
throw e;
}
Map<String, Float> map = new HashMap<String, Float>();
Map<String, Float> map = new HashMap<String, Float>();
for (ViewRow row : viewResult) {
logger.trace("ViewRow row:{}",row.toString());
JsonObject jsnobject = (JsonObject) row.value();
JSONObject objJson = new JSONObject(jsnobject.toString());
for (ViewRow row : viewResult) {
logger.trace("ViewRow row:{}",row.toString());
JsonObject jsnobject = (JsonObject) row.value();
JSONObject objJson = new JSONObject(jsnobject.toString());
Iterator<?> iterateJosn = objJson.keys();
while( iterateJosn.hasNext() ) {
String key = (String)iterateJosn.next();
Iterator<?> iterateJosn = objJson.keys();
while( iterateJosn.hasNext() ) {
String key = (String)iterateJosn.next();
Float valuetmp=Float.parseFloat(objJson.get(key).toString());
Float valuetmp=Float.parseFloat(objJson.get(key).toString());
//logger.debug("totalFilters.getClass().getSimpleName():{} UsageStorageValue.class.getSimpleName():{}", totalFilters.getClass().getSimpleName(),UsageStorageValue.class.getSimpleName());
//logger.debug("totalFilters.getClass().getSimpleName():{} UsageStorageValue.class.getSimpleName():{}", totalFilters.getClass().getSimpleName(),UsageStorageValue.class.getSimpleName());
if (key.equals("operationCount") || key.equals("dataVolume")){
if (key.equals("operationCount") || key.equals("dataVolume")){
if (map.containsKey(key)) {
map.put(key, valuetmp + map.get(key));
if (map.containsKey(key)) {
map.put(key, valuetmp + map.get(key));
// TODO verify a better method
if (totalFilters instanceof UsageStorageValue){
logger.debug("storageUsageRecord -designDocId:{}",designDocId);
if (key.equals("dataVolume")){
keyOrderingProperty=key;
totalQuota+=totalFilters.getD()+valuetmp.doubleValue();
}
}
else{
logger.debug("?UsageRecord -designDocId:{}",designDocId);
// TODO verify a better method
if (totalFilters instanceof UsageStorageValue){
logger.debug("storageUsageRecord -designDocId:{}",designDocId);
if (key.equals("dataVolume")){
keyOrderingProperty=key;
totalQuota+=totalFilters.getD()+valuetmp.doubleValue();
}
}
else{
map.put(key, valuetmp);
// TODO verify a better method
if (totalFilters instanceof UsageStorageValue){
logger.debug("storageUsageRecord -designDocId:{}",designDocId);
if (key.equals("dataVolume")){
keyOrderingProperty=key;
totalQuota+=valuetmp.doubleValue();
}
}
else{
logger.debug("?UsageRecord -designDocId:{}",designDocId);
logger.debug("?UsageRecord -designDocId:{}",designDocId);
keyOrderingProperty=key;
totalQuota+=totalFilters.getD()+valuetmp.doubleValue();
}
}
else{
map.put(key, valuetmp);
// TODO verify a better method
if (totalFilters instanceof UsageStorageValue){
logger.debug("storageUsageRecord -designDocId:{}",designDocId);
if (key.equals("dataVolume")){
keyOrderingProperty=key;
totalQuota+=valuetmp.doubleValue();
totalQuota+=valuetmp.doubleValue();
}
}
else{
logger.debug("?UsageRecord -designDocId:{}",designDocId);
keyOrderingProperty=key;
totalQuota+=valuetmp.doubleValue();
}
}
}
}
//i++;
}
//i++;
//} while (i <countFilters );
//convert usage from byte to Mb
if (totalFilters instanceof UsageStorageValue){

@ -285,21 +285,22 @@ public class AccountingPersistenceQueryCouchBaseTest {
Calendar startTimeStorage = Calendar.getInstance();
startTimeStorage.set(2015, Calendar.MAY, 1);
Calendar endTimeStorage = Calendar.getInstance();
endTimeStorage.set(2016, Calendar.DECEMBER, 18);
TemporalConstraint temporalConstraintStorage =new TemporalConstraint(startTimeStorage.getTimeInMillis(),
endTimeStorage.getTimeInMillis(), AggregationMode.DAILY);
UsageValue totalfilterStorageStatusPeriod=new UsageStorageValue(context,"name.surname",AggregatedStorageStatusRecord.class,temporalConstraintStorage);
endTimeStorage.set(2017, Calendar.FEBRUARY, 21);
/*TemporalConstraint temporalConstraintStorage =new TemporalConstraint(startTimeStorage.getTimeInMillis(),
endTimeStorage.getTimeInMillis(), AggregationMode.DAILY);*/
TemporalConstraint temporalConstraintStorage =null;
UsageValue totalfilterStorageStatusPeriod=new UsageStorageValue(context,"alessandro.pieve",AggregatedStorageStatusRecord.class,temporalConstraintStorage);
//richiedo la lista di dati usati totali
List<UsageValue> listTotalFilter=new ArrayList<UsageValue>();
listTotalFilter.add(totalfilter);
/*listTotalFilter.add(totalfilter);
listTotalFilter.add(totalfilterStorageStatus);
listTotalFilter.add(totalfilterStorageStatus_2);
listTotalFilter.add(totalfilterStorageStatus_3);
listTotalFilter.add(totalfilterStorageStatus_4);
listTotalFilter.add(totalfilterStorageStatus_4);*/
listTotalFilter.add(totalfilterStorageStatusPeriod);