diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java b/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java index a9639ef..28692fb 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBase.java @@ -44,7 +44,6 @@ import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; -import org.gcube.common.scope.api.ScopeProvider; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.RecordUtility; import org.json.JSONArray; @@ -79,12 +78,12 @@ import com.couchbase.client.java.view.ViewRow; * */ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenceBackendQuery { - + private static final Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryCouchBase.class); - + public static final String URL_PROPERTY_KEY = AccountingPersistenceConfiguration.URL_PROPERTY_KEY; public static final String PASSWORD_PROPERTY_KEY = AccountingPersistenceConfiguration.PASSWORD_PROPERTY_KEY; - + /* The environment configuration */ protected static final CouchbaseEnvironment ENV; public static final long MAX_REQUEST_LIFE_TIME = TimeUnit.MINUTES.toMillis(2); @@ -93,39 +92,39 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc public static final long VIEW_TIMEOUT_BUCKET = TimeUnit.MINUTES.toMillis(2); public static final long CONNECTION_TIMEOUT_BUCKET = TimeUnit.SECONDS.toMillis(15); public static final long CONNECTION_TIMEOUT = TimeUnit.SECONDS.toMillis(15); - + protected AccountingPersistenceBackendQueryConfiguration configuration; - + protected Cluster cluster; - - protected Map connectionMap; - + + protected Map connectionMap; + public static final String DESIGN_DOC_ID_LIST_USAGE = "ListUsage"; protected static final String MAP_REDUCE__DESIGN = ""; protected static final String MAP_REDUCE_ALL = "all"; // Used in the name of map reduce to separate keys used as filter protected static final String KEYS_SEPARATOR = "__"; - + protected static final String DESIGN_DOC_ID = "top_"; - + static { ENV = DefaultCouchbaseEnvironment.builder().connectTimeout(CONNECTION_TIMEOUT) .maxRequestLifetime(MAX_REQUEST_LIFE_TIME).queryTimeout(CONNECTION_TIMEOUT) .viewTimeout(VIEW_TIMEOUT_BUCKET).keepAliveInterval(KEEP_ALIVE_INTERVAL).kvTimeout(5000) .autoreleaseAfter(AUTO_RELEASE_AFTER).build(); - + // One Record per package is enough RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage()); RecordUtility.addRecordPackage(AggregatedServiceUsageRecord.class.getPackage()); - + } - + @Override public boolean isConnectionActive() throws Exception { return !connectionMap.values().iterator().next().isClosed(); - + } - + @Override public void prepareConnection(AccountingPersistenceBackendQueryConfiguration configuration) throws Exception { String url = configuration.getProperty(URL_PROPERTY_KEY); @@ -133,33 +132,33 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc this.configuration = configuration; this.connectionMap = new HashMap<>(); } - + protected Bucket getBucket(Class clz) throws Exception { - + UsageRecord instance = clz.newInstance(); String recordType = instance.getRecordType(); - + return getBucket(recordType); } protected static final String AGGREGATED_PREFIX = "Aggregated"; protected Bucket getBucket(String recordType) throws Exception { - if(recordType.startsWith(AGGREGATED_PREFIX)){ + if(recordType.startsWith(AGGREGATED_PREFIX)) { recordType = recordType.replace(AGGREGATED_PREFIX, ""); } Bucket bucket = connectionMap.get(recordType); - - if (bucket == null) { + + if(bucket == null) { logger.debug("Trying to get the Bucket for {}", recordType); String bucketName = configuration.getProperty(recordType); logger.debug("Bucket for {} is {}. Going to open it.", recordType, bucketName); - + bucket = cluster.openBucket(bucketName, configuration.getProperty(PASSWORD_PROPERTY_KEY)); this.connectionMap.put(recordType, bucket); } - + return bucket; } @@ -170,11 +169,11 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc public void close() throws Exception { cluster.disconnect(); } - + protected Calendar getCalendar(JSONObject obj, AggregationMode aggregationMode) throws NumberFormatException, JSONException { long millis; - if (obj.has(AggregatedRecord.START_TIME)) { + if(obj.has(AggregatedRecord.START_TIME)) { millis = new Long(obj.getString(AggregatedRecord.START_TIME)); logger.trace("The result {} was from an aggregated record. Using {}", obj.toString(), AggregatedRecord.START_TIME); @@ -186,7 +185,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc logger.trace("{} has been aligned to {}", millis, calendar.getTimeInMillis()); return calendar; } - + /* * * IS NOT USED * @@ -201,7 +200,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc TemporalConstraint temporalConstraint, List filters) throws Exception { String currentScope = ScopeProvider.instance.get(); String recordType = clz.newInstance().getRecordType(); - + Expression expression = x(BasicUsageRecord.SCOPE).eq(s(currentScope)); expression = expression.and(x(BasicUsageRecord.RECORD_TYPE).eq(s(recordType))); long startTime = temporalConstraint.getAlignedStartTime().getTimeInMillis(); @@ -210,7 +209,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc long endTime = temporalConstraint.getAlignedEndTime().getTimeInMillis(); expression = expression.and(x(AggregatedRecord.END_TIME).lt(endTime)) .or(x(AggregatedRecord.CREATION_TIME).lt(endTime)); - + AggregationMode aggregationMode = temporalConstraint.getAggregationMode(); // TODO Aggregate Results if (filters != null) { @@ -218,21 +217,21 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc expression = expression.and(x(filter.getKey()).eq(s(filter.getValue()))); } } - + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); - + GroupByPath groupByPath = select("*").from(bucket.name()).where(expression); Map map = new HashMap(); - + N1qlQueryResult result = bucket.query(groupByPath); if (!result.finalSuccess()) { logger.debug("{} failed : {}", N1qlQueryResult.class.getSimpleName(), result.errors()); return map; } - + List rows = result.allRows(); - + for (N1qlQueryRow row : rows) { try { logger.trace("Row : {}", row.toString()); @@ -241,7 +240,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc String recordString = jsonObject.toMap().toString(); logger.trace("Record String : {}", recordString); Record record = RecordUtility.getRecord(recordString); - + JSONObject obj = new JSONObject(jsonObject.toString()); Calendar calendar = getCalendar(obj, aggregationMode); if (map.containsKey(calendar)) { @@ -254,40 +253,40 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc } catch (Exception e) { logger.warn("Unable to eleborate result for {}", row.toString()); } - + logger.trace("\n\n\n"); } - + return map; } */ - + protected Calendar getCalendarFromArray(JsonArray array) throws JSONException { boolean startFound = false; Calendar calendar = Calendar.getInstance(TemporalConstraint.DEFAULT_TIME_ZONE); int count = 0; CalendarEnum[] calendarValues = CalendarEnum.values(); - for (int i = 0; i < array.size(); i++) { + for(int i = 0; i < array.size(); i++) { try { int value = array.getInt(i); int calendarValue = calendarValues[count].getCalendarValue(); - if (calendarValue == Calendar.MONTH) { + if(calendarValue == Calendar.MONTH) { value--; } calendar.set(calendarValue, value); count++; startFound = true; - } catch (Exception e) { + } catch(Exception e) { // logger.trace("The provide value is not an int. {}", // array.get(i).toString()); - if (startFound) { + if(startFound) { break; } - + } } - for (int j = count; j < calendarValues.length; j++) { - if (calendarValues[j].getCalendarValue() == Calendar.DAY_OF_MONTH) { + for(int j = count; j < calendarValues.length; j++) { + if(calendarValues[j].getCalendarValue() == Calendar.DAY_OF_MONTH) { calendar.set(calendarValues[j].getCalendarValue(), 1); } else { calendar.set(calendarValues[j].getCalendarValue(), 0); @@ -295,54 +294,54 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc } return calendar; } - + protected JsonArray getRangeKey(long time, AggregationMode aggregationMode, boolean wildCard, boolean endKey) throws JSONException { - + JsonArray array = JsonArray.create(); Calendar calendar = Calendar.getInstance(); calendar.setTimeInMillis(time); CalendarEnum[] values = CalendarEnum.values(); - if (endKey) { + if(endKey) { calendar.add(values[aggregationMode.ordinal()].getCalendarValue(), 1); } - for (int i = 0; i <= aggregationMode.ordinal(); i++) { + for(int i = 0; i <= aggregationMode.ordinal(); i++) { int value = calendar.get(values[i].getCalendarValue()); - if (values[i].getCalendarValue() == Calendar.MONTH) { + if(values[i].getCalendarValue() == Calendar.MONTH) { value = value + 1; } array.add(value); } - if (wildCard) { + if(wildCard) { array.add("{}"); } return array; } - + // OLD METHOD OF DIVISION MAP_REDUCE DESIGN @Deprecated - protected String getDesignDocId(Class> recordClass) + protected String getDesignDocId(Class> recordClass) throws InstantiationException, IllegalAccessException { return String.format("%s%s", MAP_REDUCE__DESIGN, recordClass.newInstance().getRecordType()); } - - protected String getDesignDocIdSpecific(Class> recordClass, - Collection keys) throws InstantiationException, IllegalAccessException { + + protected String getDesignDocIdSpecific(Class> recordClass, Collection keys) + throws InstantiationException, IllegalAccessException { String specific = "all"; - if (!keys.isEmpty()) { + if(!keys.isEmpty()) { specific = keys.iterator().next(); } String getDesigndocIdSpecific = specific; // logger.trace("Use a designDocIDSpecific:{}",getDesigndocIdSpecific); return getDesigndocIdSpecific; } - + public static String getMapReduceFunctionName(Collection collection) { String reduceFunction = MAP_REDUCE_ALL; - if (!collection.isEmpty()) { + if(!collection.isEmpty()) { reduceFunction = null; - for (String property : collection) { - if (reduceFunction == null) { + for(String property : collection) { + if(reduceFunction == null) { reduceFunction = property; } else { reduceFunction = reduceFunction + KEYS_SEPARATOR + property; @@ -351,19 +350,19 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc } return reduceFunction; } - + public static String getMapReduceFunctionNameTopMap(String top, Collection collection) { logger.debug("top:{}", top); logger.debug("collection:{}", collection.toString()); - + String reduceFunction = MAP_REDUCE_ALL; - - if (!collection.isEmpty()) { + + if(!collection.isEmpty()) { reduceFunction = top; - for (String property : collection) { - if (!property.equals(top)) { - - if (reduceFunction == null) { + for(String property : collection) { + if(!property.equals(top)) { + + if(reduceFunction == null) { reduceFunction = property; } else { reduceFunction = reduceFunction + KEYS_SEPARATOR + property; @@ -373,7 +372,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc } return reduceFunction; } - + /** * EXPERIMENTAL DEPRECATED generate a name of design doc id for a top * @@ -382,15 +381,15 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc */ public static String getDesignDocIdName(Collection collection) { String reduceFunction = MAP_REDUCE_ALL; - if (!collection.isEmpty()) { + if(!collection.isEmpty()) { reduceFunction = null; String property = collection.iterator().next(); reduceFunction = property; - + } return reduceFunction; } - + /* * EXPERIMENTAL * @@ -480,35 +479,36 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc * * } */ - - protected SortedMap mapReduceQuery(Class> clz, + + protected SortedMap mapReduceQuery(Class> clz, TemporalConstraint temporalConstraint, List filters, String context, Boolean valueEmpty, Boolean noScope) throws Exception { - String currentScope = null; - if (context == null) - currentScope = ScopeProvider.instance.get(); - else + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); + + if(context != null) { currentScope = context; + } + JsonArray startKey = JsonArray.create(); - + JsonArray endKey = JsonArray.create(); // no scope call a map reduce without scope in startkey and endkey - if (!noScope) { + if(!noScope) { startKey.add(currentScope); endKey.add(currentScope); } - + AggregationMode aggregationMode = temporalConstraint.getAggregationMode(); - + JsonArray temporalStartKey = getRangeKey(temporalConstraint.getStartTime(), aggregationMode, false, false); - + JsonArray temporalEndKey = getRangeKey(temporalConstraint.getEndTime(), aggregationMode, false, false); - + Set recordKeysSet = AccountingPersistenceQuery.getQuerableKeys(clz); - + Collection keys = new TreeSet<>(); - - if (filters != null && filters.size() != 0) { + + if(filters != null && filters.size() != 0) { // Sorting filter for call a mapreduce Collections.sort(filters, new Comparator() { @Override @@ -517,13 +517,13 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc return result; } }); - for (Filter filter : filters) { + for(Filter filter : filters) { String filterKey = filter.getKey(); String filterValue = filter.getValue(); - - if (filterKey != null && filterKey.compareTo("") != 0 && recordKeysSet.contains(filterKey)) { - if (filterValue != null && filterValue.compareTo("") != 0) { - if (keys.contains(filterKey)) { + + if(filterKey != null && filterKey.compareTo("") != 0 && recordKeysSet.contains(filterKey)) { + if(filterValue != null && filterValue.compareTo("") != 0) { + if(keys.contains(filterKey)) { throw new DuplicatedKeyFilterException("Only one value per Filter key is allowed"); } startKey.add(filterValue); @@ -533,37 +533,37 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc throw new KeyException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } - + } else { throw new ValueException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } } } - + // +1 because mode start from 0 // +1 because of scope at the beginning int scopeDateGroupLevel = aggregationMode.ordinal() + 1 + 1; int groupLevel = scopeDateGroupLevel; - if (filters != null) { + if(filters != null) { groupLevel += keys.size(); } // String designDocId = getDesignDocId(clz); String designDocId = getDesignDocIdSpecific(clz, keys); - if (noScope) { + if(noScope) { designDocId = "noContext"; groupLevel = groupLevel - 1; } // logger.trace("designDocIdNew :{}",designDocId); - for (Object temporal : temporalStartKey.toList()) { - if (!temporal.toString().isEmpty()) + for(Object temporal : temporalStartKey.toList()) { + if(!temporal.toString().isEmpty()) startKey.add(temporal); } int count = 1; - for (Object temporal : temporalEndKey.toList()) { - if (!temporal.toString().isEmpty()) { + for(Object temporal : temporalEndKey.toList()) { + if(!temporal.toString().isEmpty()) { // couchbase exclude last value - if (count == temporalEndKey.size()) + if(count == temporalEndKey.size()) temporal = (int) temporal + 1; endKey.add(temporal); } @@ -576,41 +576,41 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc query.startKey(startKey); query.endKey(endKey); query.descending(false); - + logger.trace( "Bucket :{}, Design Doc ID : {}, View Name : {}, " + "Group Level : {}, Start Key : {}, End Key : {}," + "temporalStartKey :{}, temporalEndKey :{}", clz.getSimpleName(), designDocId, viewName, groupLevel, startKey, endKey, temporalStartKey.toString(), temporalEndKey.toString()); - SortedMap infos = new TreeMap<>(); - + SortedMap infos = new TreeMap<>(); + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); - + ViewResult viewResult; try { // execute query in a specify bucket viewResult = bucket.query(query); - - } catch (Exception e) { + + } catch(Exception e) { logger.error(e.getLocalizedMessage()); throw e; } - - for (ViewRow row : viewResult) { - + + for(ViewRow row : viewResult) { + JsonArray array = (JsonArray) row.key(); Calendar calendar = getCalendarFromArray(array); JsonObject value = (JsonObject) row.value(); JSONObject obj = new JSONObject(value.toString()); Info info = new Info(calendar, obj); infos.put(calendar, info); - + } logger.trace("valueEmpty not permitted:{}", valueEmpty); // infos not empity is permitted only for getTimeSeries and Top - if (valueEmpty) { - if (infos.isEmpty()) { + if(valueEmpty) { + if(infos.isEmpty()) { logger.trace("infos is empity"); query = ViewQuery.from(designDocId, viewName); query.groupLevel(groupLevel); @@ -618,22 +618,22 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc try { // execute query in a specify bucket viewResult = bucket.query(query); - } catch (Exception e) { + } catch(Exception e) { logger.warn("not execute query", e.getLocalizedMessage()); // throw e; } - + try { - if (viewResult.totalRows() != 0) { + if(viewResult.totalRows() != 0) { ViewRow row = viewResult.allRows().get(0); JsonArray array = getRangeKey(temporalConstraint.getStartTime(), aggregationMode, false, false); Calendar calendar = getCalendarFromArray(array); - + JsonObject value = (JsonObject) row.value(); JSONObject objJson = new JSONObject(value.toString()); JSONObject objJsontemplate = new JSONObject(); Iterator iterateJson = objJson.keys(); - while (iterateJson.hasNext()) { + while(iterateJson.hasNext()) { String key = (String) iterateJson.next(); objJsontemplate.put(key, 0); } @@ -641,8 +641,8 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc Info info = new Info(calendar, objJsontemplate); infos.put(calendar, info); } - - } catch (Exception e) { + + } catch(Exception e) { logger.warn("error :{}", e.getLocalizedMessage()); } } @@ -650,48 +650,48 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc logger.trace("infos:{}", infos.toString()); return infos; } - + @Override - public SortedMap getTimeSeries(Class> clz, + public SortedMap getTimeSeries(Class> clz, TemporalConstraint temporalConstraint, List filters) throws Exception { - - SortedMap map = mapReduceQuery(clz, temporalConstraint, filters, null, true, false); + + SortedMap map = mapReduceQuery(clz, temporalConstraint, filters, null, true, false); return map; } - + @Override - public SortedMap getNoContextTimeSeries(Class> clz, + public SortedMap getNoContextTimeSeries(Class> clz, TemporalConstraint temporalConstraint, List filters) throws Exception { - - SortedMap map = mapReduceQuery(clz, temporalConstraint, filters, null, true, true); + + SortedMap map = mapReduceQuery(clz, temporalConstraint, filters, null, true, true); return map; } - + @Override - public SortedMap> getTopValues( - Class> clz, TemporalConstraint temporalConstraint, List filters, - String topKey, String orderingProperty) throws Exception { - + public SortedMap> getTopValues(Class> clz, + TemporalConstraint temporalConstraint, List filters, String topKey, String orderingProperty) + throws Exception { + Comparator comparator = new Comparator() { @Override public int compare(NumberedFilter o1, NumberedFilter o2) { int result = -o1.compareTo(o2); - - if (result == 0) { + + if(result == 0) { result = o1.compareTo((Filter) o2); } - + return result; } }; - SortedMap> ret = new TreeMap<>(comparator); - + SortedMap> ret = new TreeMap<>(comparator); + SortedSet top = null; - + // top = getNextPossibleValues(clz,temporalConstraint, filters, topKey, // orderingProperty); - - if (usingNextPossibleValuesWithMap(clz, topKey, filters)) { + + if(usingNextPossibleValuesWithMap(clz, topKey, filters)) { logger.trace("getNextPossibleValues using map"); top = getNextPossibleValuesWithMap(clz, temporalConstraint, filters, topKey, orderingProperty); } else { @@ -699,17 +699,17 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc top = getNextPossibleValues(clz, temporalConstraint, filters, topKey, orderingProperty); } logger.trace("getNextPossibleValues:{}", top.toString()); - for (NumberedFilter nf : top) { + for(NumberedFilter nf : top) { filters.add(nf); - SortedMap map = mapReduceQuery(clz, temporalConstraint, filters, null, true, false); + SortedMap map = mapReduceQuery(clz, temporalConstraint, filters, null, true, false); ret.put(nf, map); filters.remove(nf); } - + return ret; - + } - + /** * SPERIMENTAL Used for verify if have exist map for calculate a top * @@ -718,16 +718,16 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc * @return boolean * @throws Exception */ - protected boolean usingNextPossibleValuesWithMap(Class> clz, String topKey, + protected boolean usingNextPossibleValuesWithMap(Class> clz, String topKey, List filters) throws Exception { - + logger.debug("usingNextPossibleValuesWithMap init"); Collection keys = new TreeSet<>(); Set recordKeysSet; try { recordKeysSet = AccountingPersistenceQuery.getQuerableKeys(clz); keys.add(topKey); - if (filters != null && filters.size() != 0) { + if(filters != null && filters.size() != 0) { // Sorting filter for call a mapreduce Collections.sort(filters, new Comparator() { @Override @@ -736,13 +736,13 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc return result; } }); - for (Filter filter : filters) { + for(Filter filter : filters) { String filterKey = filter.getKey(); String filterValue = filter.getValue(); - - if (filterKey != null && filterKey.compareTo("") != 0 && recordKeysSet.contains(filterKey)) { - if (filterValue != null && filterValue.compareTo("") != 0) { - if (keys.contains(filterKey)) { + + if(filterKey != null && filterKey.compareTo("") != 0 && recordKeysSet.contains(filterKey)) { + if(filterValue != null && filterValue.compareTo("") != 0) { + if(keys.contains(filterKey)) { throw new DuplicatedKeyFilterException("Only one value per Filter key is allowed"); } keys.add(filterKey); @@ -750,14 +750,14 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc throw new KeyException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } - + } else { throw new ValueException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } } } - } catch (Exception e1) { + } catch(Exception e1) { logger.warn("usingNextPossibleValuesWithMap -exception with filter:{}", filters.toString()); return false; } @@ -765,22 +765,22 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc String viewName = getMapReduceFunctionNameTopMap(topKey, keys); // String designDocId =DESIGN_DOC_ID+getDesignDocIdName(keys); String designDocId = DESIGN_DOC_ID + topKey; - + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); BucketManager bucketManager = bucket.bucketManager(); // logger.debug("----"+bucketManager.getDesignDocument(designDocId)); - - if (bucketManager.getDesignDocument(designDocId) != null) { + + if(bucketManager.getDesignDocument(designDocId) != null) { logger.debug("usingNextPossibleValuesWithMap designDocId exist:{}-and viewname:{}-", designDocId, viewName); - for (View view : bucketManager.getDesignDocument(designDocId).views()) { + for(View view : bucketManager.getDesignDocument(designDocId).views()) { logger.debug("found:{}- ", view.name()); - if (view.name().equals(viewName)) { + if(view.name().equals(viewName)) { logger.debug("usingNextPossibleValuesWithMap viewname exist"); return true; } } - + } else { logger.debug("usingNextPossibleValuesWithQuery"); return false; @@ -794,7 +794,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc */ return false; } - + /** * Calculate a next possible value with map (faster but with greater demand * for resources) @@ -807,34 +807,34 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc * @return SortedSet * @throws Exception */ - public SortedSet getNextPossibleValuesWithMap(Class> clz, + public SortedSet getNextPossibleValuesWithMap(Class> clz, TemporalConstraint temporalConstraint, List filters, String key, String orderingProperty) throws Exception { - + logger.debug("getNextPossibleValuesWithMap init"); - String currentScope = ScopeProvider.instance.get(); - - if (orderingProperty == null) { + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); + + if(orderingProperty == null) { orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(clz); } - + JsonArray startKey = JsonArray.create(); startKey.add(currentScope); JsonArray endKey = JsonArray.create(); endKey.add(currentScope); - + AggregationMode aggregationMode = temporalConstraint.getAggregationMode(); - + JsonArray temporalStartKey = getRangeKey(temporalConstraint.getStartTime(), aggregationMode, false, false); - + JsonArray temporalEndKey = getRangeKey(temporalConstraint.getEndTime(), aggregationMode, false, false); - + Set recordKeysSet = AccountingPersistenceQuery.getQuerableKeys(clz); - + Collection keys = new TreeSet<>(); - + keys.add(key); - + /* BEGIN DELETE ONLY FOR TEST */ Collection selectExpressions = new ArrayList<>(); selectExpressions.add(x("SUM(CASE WHEN " + getSpecializedProperty(clz, orderingProperty) + " IS NOT NULL THEN " @@ -850,15 +850,15 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc Expression[] selectExpressionArray = new Expression[selectExpressions.size()]; selectExpressions.toArray(selectExpressionArray); Sort sort = Sort.desc(orderingProperty); - + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); - + OffsetPath path = select(selectExpressionArray).from(bucket.name()).where(whereExpression).groupBy(key) .orderBy(sort); - + /* END DELETE ONLY FOR TEST */ - if (filters != null && filters.size() != 0) { + if(filters != null && filters.size() != 0) { // Sorting filter for call a mapreduce Collections.sort(filters, new Comparator() { @Override @@ -867,13 +867,13 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc return result; } }); - for (Filter filter : filters) { + for(Filter filter : filters) { String filterKey = filter.getKey(); String filterValue = filter.getValue(); - - if (filterKey != null && filterKey.compareTo("") != 0 && recordKeysSet.contains(filterKey)) { - if (filterValue != null && filterValue.compareTo("") != 0) { - if (keys.contains(filterKey)) { + + if(filterKey != null && filterKey.compareTo("") != 0 && recordKeysSet.contains(filterKey)) { + if(filterValue != null && filterValue.compareTo("") != 0) { + if(keys.contains(filterKey)) { throw new DuplicatedKeyFilterException("Only one value per Filter key is allowed"); } startKey.add(filterValue); @@ -885,27 +885,27 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc throw new KeyException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } - + } else { throw new ValueException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } } } - + logger.debug("Alternative Query for top:" + path.toString()); - + int groupLevel = 1; - - for (Object temporal : temporalStartKey.toList()) { - if (!temporal.toString().isEmpty()) + + for(Object temporal : temporalStartKey.toList()) { + if(!temporal.toString().isEmpty()) startKey.add(temporal); } int count = 1; - for (Object temporal : temporalEndKey.toList()) { - if (!temporal.toString().isEmpty()) { + for(Object temporal : temporalEndKey.toList()) { + if(!temporal.toString().isEmpty()) { // couchbase exclude last value - if (count == temporalEndKey.size()) + if(count == temporalEndKey.size()) temporal = (int) temporal + 1; endKey.add(temporal); } @@ -926,46 +926,46 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc + "temporalStartKey :{}, temporalEndKey :{}", clz.getSimpleName(), designDocId, viewName, groupLevel, startKey, endKey, temporalStartKey.toString(), temporalEndKey.toString()); - + Comparator comparator = new Comparator() { @Override public int compare(NumberedFilter o1, NumberedFilter o2) { int compareResult = -o1.compareTo(o2); - if (compareResult == 0) { + if(compareResult == 0) { compareResult = 1; } return compareResult; } }; SortedSet ret = new TreeSet<>(comparator); - + ViewResult viewResult; try { // execute query in a specify bucket viewResult = bucket.query(query); - - } catch (Exception e) { + + } catch(Exception e) { logger.error(e.getLocalizedMessage()); throw e; } - + ViewRow row = viewResult.allRows().get(0); - + JsonObject value = (JsonObject) row.value(); JSONObject objectValueTop = new JSONObject(value.toString()); Iterator iterateJosn = objectValueTop.keys(); - while (iterateJosn.hasNext()) { + while(iterateJosn.hasNext()) { String keyTop = (String) iterateJosn.next(); Number n = (Number) objectValueTop.get(keyTop); - if (n == null) + if(n == null) n = 0; NumberedFilter numberedFilter = new NumberedFilter(key, keyTop, n, orderingProperty); ret.add(numberedFilter); } return ret; - + } - + /** * Calculate a next possible value with query (more slow but with fewer * resources ) @@ -979,49 +979,48 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc * @throws Exception */ @Override - public SortedSet getNextPossibleValues(Class> clz, + public SortedSet getNextPossibleValues(Class> clz, TemporalConstraint temporalConstraint, List filters, String key, String orderingProperty) throws Exception { - - String currentScope = ScopeProvider.instance.get(); - // String recordType = clz.newInstance().getRecordType(); - - if (orderingProperty == null) { + + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); + + if(orderingProperty == null) { orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(clz); } - + Collection selectExpressions = new ArrayList<>(); // add select expression and check if exist selectExpressions.add(x("SUM(CASE WHEN " + getSpecializedProperty(clz, orderingProperty) + " IS NOT NULL THEN " + getSpecializedProperty(clz, orderingProperty) + " ELSE 1 END )").as(orderingProperty)); selectExpressions.add(x("(CASE WHEN " + getSpecializedProperty(clz, key) + " IS NOT NULL THEN " + getSpecializedProperty(clz, key) + " ELSE 'UNKNOWN' END )").as(key)); - + // add where expression Expression whereExpression = x(getSpecializedProperty(clz, BasicUsageRecord.SCOPE)).eq(s(currentScope)); - + long startTime = temporalConstraint.getAlignedStartTime().getTimeInMillis(); whereExpression = whereExpression .and(x(getSpecializedProperty(clz, AggregatedRecord.START_TIME)).gt(startTime)); long endTime = temporalConstraint.getEndTime(); - + whereExpression = whereExpression.and(x(getSpecializedProperty(clz, AggregatedRecord.END_TIME)).lt(endTime)); - + Set recordKeysSet = AccountingPersistenceQuery.getQuerableKeys(clz); - + // list filter used for remove duplicate filter Collection keys = new TreeSet<>(); - - if (filters != null && filters.size() != 0) { - for (Filter filter : filters) { - + + if(filters != null && filters.size() != 0) { + for(Filter filter : filters) { + String filterKey = filter.getKey(); String filterValue = filter.getValue(); - - if (filterKey != null && filterKey.compareTo("") != 0 && recordKeysSet.contains(filterKey)) { - - if (filterValue != null && filterValue.compareTo("") != 0) { - if (keys.contains(filterKey)) { + + if(filterKey != null && filterKey.compareTo("") != 0 && recordKeysSet.contains(filterKey)) { + + if(filterValue != null && filterValue.compareTo("") != 0) { + if(keys.contains(filterKey)) { throw new DuplicatedKeyFilterException("Only one value per Filter key is allowed"); } whereExpression = whereExpression @@ -1031,32 +1030,32 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc throw new KeyException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } - + } else { throw new ValueException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } - + } } Expression[] selectExpressionArray = new Expression[selectExpressions.size()]; selectExpressions.toArray(selectExpressionArray); - + Sort sort = Sort.desc(orderingProperty); - + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); - + OffsetPath path = select(selectExpressionArray).from(bucket.name()).where(whereExpression).groupBy(key) .orderBy(sort); - + logger.debug("Query for top:" + path.toString()); - + Comparator comparator = new Comparator() { @Override public int compare(NumberedFilter o1, NumberedFilter o2) { int compareResult = -o1.compareTo(o2); - if (compareResult == 0) { + if(compareResult == 0) { compareResult = 1; } return compareResult; @@ -1064,15 +1063,15 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc }; SortedSet ret = new TreeSet<>(comparator); N1qlQueryResult result = bucket.query(path); - - if (!result.finalSuccess()) { + + if(!result.finalSuccess()) { logger.debug("{} failed : {}", N1qlQueryResult.class.getSimpleName(), result.errors()); throw new Exception("Query Failed :\n" + result.errors()); } - + List rows = result.allRows(); - - for (N1qlQueryRow row : rows) { + + for(N1qlQueryRow row : rows) { try { JsonObject jsonObject = row.value(); // logger.trace("JsonObject : {}", row.value()+" key"+key); @@ -1080,25 +1079,25 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc // verify for a not null value String value = jsonObject.getString(key); Number n = jsonObject.getDouble(orderingProperty); - if (n == null) + if(n == null) n = 0; // logger.trace("pre:{}, key:{}, value:{}, // n:{},orderingProperty:{}",jsonObject.toString(),key, value, // n, orderingProperty); - + NumberedFilter numberedFilter = new NumberedFilter(key, value, n, orderingProperty); - + ret.add(numberedFilter); - - } catch (Exception e) { + + } catch(Exception e) { logger.warn("Unable to eleborate result for {}", row.toString()); e.printStackTrace(); } - + } return ret; } - + /** * Return a list of context time series (used for portlet accounting * context) @@ -1111,37 +1110,36 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc * @throws Exception */ @Override - public SortedMap> getContextTimeSeries( - Class> clz, TemporalConstraint temporalConstraint, List filters, - List contexts) throws Exception { + public SortedMap> getContextTimeSeries(Class> clz, + TemporalConstraint temporalConstraint, List filters, List contexts) throws Exception { logger.trace("getContextTimeSeries for contexts:{}", contexts.toString()); SortedSet listContexts = new TreeSet(); - for (String context : contexts) { + for(String context : contexts) { Filter contextLabel = new Filter("context", context); listContexts.add(contextLabel); } - SortedMap> ret = new TreeMap<>(); - for (Filter nf : listContexts) { + SortedMap> ret = new TreeMap<>(); + for(Filter nf : listContexts) { logger.debug("detail time series :{}", nf.toString()); - SortedMap map = mapReduceQuery(clz, temporalConstraint, filters, nf.getValue(), false, + SortedMap map = mapReduceQuery(clz, temporalConstraint, filters, nf.getValue(), false, false); - if (!map.isEmpty()) { + if(!map.isEmpty()) { ret.put(nf, map); } - + filters.remove(nf); } return ret; } - + protected String getQualifiedProperty(String property) { // DEVELOPING return (property); - + } - + // Use for property into a specify bucket - protected String getSpecializedProperty(Class> clz, String property) + protected String getSpecializedProperty(Class> clz, String property) throws Exception { @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); @@ -1150,11 +1148,11 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc @Override @Deprecated - public SortedSet getFilterValues(Class> clz, + public SortedSet getFilterValues(Class> clz, TemporalConstraint temporalConstraint, List filters, String key) throws Exception { return getFilterValues(clz, temporalConstraint, filters, key, null); } - + /** * Used for list a possible values for each filter * @@ -1166,61 +1164,61 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc * @throws Exception */ @Override - public SortedSet getFilterValues(Class> clz, + public SortedSet getFilterValues(Class> clz, TemporalConstraint temporalConstraint, List filters, String key, Integer limit) throws Exception { - - String currentScope = ScopeProvider.instance.get(); - + + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); + JsonArray startKey = JsonArray.create(); startKey.add(currentScope); - + int scopeDateGroupLevel = 2; int groupLevel = scopeDateGroupLevel; // NO ADD A SPECIFIY DESIGN DOC ID FAMILY String designDocId = getDesignDocId(clz) + "Value"; - + String viewName = key; logger.trace("designDocId:{} view:{} startKey:{} groupLevel:{}", designDocId, key, startKey, groupLevel); ViewQuery query = ViewQuery.from(designDocId, viewName); - + query.inclusiveEnd(); query.groupLevel(groupLevel); query.startKey(startKey); query.descending(false); - if(limit !=null){ + if(limit != null) { query.limit(limit); } - + String orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(clz); - + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); - + ViewResult viewResult; try { // execute query in a specify bucket viewResult = bucket.query(query); - - } catch (Exception e) { + + } catch(Exception e) { logger.error("error executing the query", e); throw e; } - + Comparator comparator = new Comparator() { - + @Override public int compare(NumberedFilter o1, NumberedFilter o2) { - if (o1.getValue() == null) + if(o1.getValue() == null) o1.setValue(""); - if (o2.getValue() == null) + if(o2.getValue() == null) o2.setValue(""); return o1.getValue().compareTo(o2.getValue()); } - + }; SortedSet ret = new TreeSet<>(comparator); - - for (ViewRow row : viewResult) { + + for(ViewRow row : viewResult) { String value = (String) row.value(); NumberedFilter numberedFilter = new NumberedFilter(key, value, 0, orderingProperty); ret.add(numberedFilter); @@ -1228,7 +1226,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc logger.trace("returning {} values", ret.size()); return ret; } - + /** * SPERIMENTAL now is not used * @@ -1239,54 +1237,54 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc * @throws Exception */ @Override - public JSONObject getUsageValue(Class> clz, TemporalConstraint temporalConstraint, + public JSONObject getUsageValue(Class> clz, TemporalConstraint temporalConstraint, Filter applicant) throws Exception { - - String currentScope = ScopeProvider.instance.get(); - + + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); + JsonArray startKey = JsonArray.create(); startKey.add(currentScope); - + JsonArray endKey = JsonArray.create(); endKey.add(currentScope); - + AggregationMode aggregationMode = temporalConstraint.getAggregationMode(); - + JsonArray temporalStartKey = getRangeKey(temporalConstraint.getStartTime(), aggregationMode, false, false); - + JsonArray temporalEndKey = getRangeKey(temporalConstraint.getEndTime(), aggregationMode, false, false); - + startKey.add(applicant.getValue()); - - for (Object temporal : temporalStartKey.toList()) { - if (!temporal.toString().isEmpty()) + + for(Object temporal : temporalStartKey.toList()) { + if(!temporal.toString().isEmpty()) startKey.add(temporal); } - + endKey.add(applicant.getValue()); - + int count = 1; - for (Object temporal : temporalEndKey.toList()) { - if (!temporal.toString().isEmpty()) { + for(Object temporal : temporalEndKey.toList()) { + if(!temporal.toString().isEmpty()) { // couchbase exclude last value - if (count == temporalEndKey.size()) + if(count == temporalEndKey.size()) temporal = (int) temporal + 1; endKey.add(temporal); } count++; } - + // +1 because mode start from 0 // +1 because have afilter value from 1 // +1 because of scope at the beginning int scopeDateGroupLevel = aggregationMode.ordinal() + 1 + 1 + 1; int groupLevel = scopeDateGroupLevel; - + Collection keys = new TreeSet<>(); keys.add(applicant.getKey()); // ADD A SPECIFIY DESIGN DOC ID FAMILY String designDocId = getDesignDocIdSpecific(clz, keys); - + String viewName = applicant.getKey(); ViewQuery query = ViewQuery.from(designDocId, viewName); query.inclusiveEnd(); @@ -1294,40 +1292,40 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc query.startKey(startKey); query.endKey(endKey); query.descending(false); - + logger.trace( "Bucket :{}, Design Doc ID : {}, View Name : {}, " + "Group Level : {}, Start Key : {}, End Key : {}," + "temporalStartKey :{}, temporalEndKey :{}", clz.getSimpleName(), designDocId, viewName, groupLevel, startKey, endKey, temporalStartKey.toString(), temporalEndKey.toString()); - + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); - + ViewResult viewResult; try { // execute query in a specify bucket viewResult = bucket.query(query); - - } catch (Exception e) { + + } catch(Exception e) { logger.error(e.getLocalizedMessage()); throw e; } - - Map map = new HashMap(); - - for (ViewRow row : viewResult) { - + + Map map = new HashMap(); + + for(ViewRow row : viewResult) { + JsonObject jsnobject = (JsonObject) row.value(); JSONObject objJson = new JSONObject(jsnobject.toString()); - + Iterator iterateJosn = objJson.keys(); - while (iterateJosn.hasNext()) { + while(iterateJosn.hasNext()) { String key = (String) iterateJosn.next(); Float valuetmp = Float.parseFloat(objJson.get(key).toString()); - - if (key.equals("operationCount") || key.equals("dataVolume")) { - if (map.containsKey(key)) { + + if(key.equals("operationCount") || key.equals("dataVolume")) { + if(map.containsKey(key)) { map.put(key, valuetmp + map.get(key)); } else map.put(key, valuetmp); @@ -1337,7 +1335,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc JSONObject result = new JSONObject(map); return result; } - + /** * * Used for calculate a usage value for each element of list (QUOTA) @@ -1348,25 +1346,25 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc */ @Override public List getUsageValueQuotaTotal(List listUsage) throws Exception { - + logger.debug("getUsageValueQuotaTotal init with list:{}", listUsage); - + String keyOrderingProperty = null; - for (UsageValue totalFilters : listUsage) { - + for(UsageValue totalFilters : listUsage) { + String currentScope = totalFilters.getContext(); - + Collection keys = new TreeSet<>(); keys.add("consumerId"); String designDocId = getDesignDocIdSpecific(totalFilters.getClz(), keys); - + JsonArray temporalStartKey = null; JsonArray temporalEndKey = null; - + logger.trace("temporalConstraint:{}", totalFilters); TemporalConstraint temporalConstraint = totalFilters.getTemporalConstraint(); - - if (temporalConstraint == (null)) { + + if(temporalConstraint == (null)) { // used for no temporalConstraint logger.trace("Not found temporalConstraint"); Calendar startTime = Calendar.getInstance(); @@ -1374,20 +1372,20 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc Calendar endTime = Calendar.getInstance(); temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); - if (totalFilters instanceof UsageStorageValue) { + if(totalFilters instanceof UsageStorageValue) { designDocId = "QuotaTotalSeparated"; } - - } else if (totalFilters.getClz().getSimpleName() + + } else if(totalFilters.getClz().getSimpleName() .equals(AggregatedStorageStatusRecord.class.getSimpleName())) { logger.trace("AggregatedStorageStatusRecord with temporalConstraint"); designDocId = "Quota"; } - + AggregationMode aggregationMode = temporalConstraint.getAggregationMode(); temporalStartKey = getRangeKey(temporalConstraint.getStartTime(), aggregationMode, false, false); temporalEndKey = getRangeKey(temporalConstraint.getEndTime(), aggregationMode, false, false); - + Double totalQuota = 0.00; /* @@ -1396,26 +1394,25 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc } */ - // do { String viewNameTmp = null; JsonArray startKeyTmp = JsonArray.create(); startKeyTmp.add(currentScope); - + JsonArray endKeyTmp = JsonArray.create(); endKeyTmp.add(currentScope); - + int groupLevelTmp = 2; // FiltersValue singleFilter=null; viewNameTmp = AggregatedServiceUsageRecord.CONSUMER_ID; startKeyTmp.add(totalFilters.getIdentifier()); endKeyTmp.add(totalFilters.getIdentifier()); - if (totalFilters instanceof UsageServiceValue) { + if(totalFilters instanceof UsageServiceValue) { // logger.debug("******UsageServiceValue"); - + UsageServiceValue totalFiltersService = (UsageServiceValue) totalFilters; // singleFilter=totalFiltersService.getFiltersValue().get(i); - for (Filter filter : totalFiltersService.getFilters()) { + for(Filter filter : totalFiltersService.getFilters()) { viewNameTmp = viewNameTmp + "__" + filter.getKey(); startKeyTmp.add(filter.getValue()); endKeyTmp.add(filter.getValue()); @@ -1423,15 +1420,15 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc } } // not defined temporal constraint - for (Object temporal : temporalStartKey.toList()) { - if (!temporal.toString().isEmpty()) + for(Object temporal : temporalStartKey.toList()) { + if(!temporal.toString().isEmpty()) startKeyTmp.add(temporal); } int count = 1; - for (Object temporal : temporalEndKey.toList()) { - if (!temporal.toString().isEmpty()) { + for(Object temporal : temporalEndKey.toList()) { + if(!temporal.toString().isEmpty()) { // couchbase excludes last value - if (count == temporalEndKey.size()) + if(count == temporalEndKey.size()) temporal = (int) temporal + 1; endKeyTmp.add(temporal); } @@ -1441,7 +1438,7 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc + "Group Level : {}, Start Key : {}, End Key : {}," + "temporalStartKey :{}, temporalEndKey :{}", totalFilters.getClz().getSimpleName(), designDocId, viewNameTmp, groupLevelTmp, startKeyTmp, endKeyTmp, temporalStartKey.toString(), temporalEndKey.toString()); - + ViewQuery query = ViewQuery.from(designDocId, viewNameTmp); query.inclusiveEnd(); query.groupLevel(groupLevelTmp); @@ -1450,40 +1447,40 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc query.descending(false); query.onError(OnError.STOP); logger.trace("query row:{}", query.toString()); - + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) totalFilters.getClz()); - + ViewResult viewResult; try { viewResult = bucket.query(query); - } catch (Exception e) { + } catch(Exception e) { logger.error(e.getLocalizedMessage()); throw e; } - + logger.trace("viewResult row:{}", viewResult.toString()); - Map map = new HashMap(); - - for (ViewRow row : viewResult) { + Map map = new HashMap(); + + for(ViewRow row : viewResult) { logger.trace("ViewRow row:{}", row.toString()); JsonObject jsnobject = (JsonObject) row.value(); JSONObject objJson = new JSONObject(jsnobject.toString()); - + Iterator iterateJosn = objJson.keys(); - while (iterateJosn.hasNext()) { + while(iterateJosn.hasNext()) { String key = (String) iterateJosn.next(); - - if (totalFilters instanceof UsageStorageValue) { + + if(totalFilters instanceof UsageStorageValue) { JSONArray valueStorage = (JSONArray) objJson.get(key); logger.debug("--storageUsageRecord -key:{} value:{}", key, valueStorage.get(0)); totalQuota += Float.parseFloat(valueStorage.get(0).toString()); keyOrderingProperty = "dataVolume"; } else { - if (key.equals("operationCount") || key.equals("dataVolume")) { + if(key.equals("operationCount") || key.equals("dataVolume")) { Float valuetmp = Float.parseFloat(objJson.get(key).toString()); - - if (map.containsKey(key)) { + + if(map.containsKey(key)) { map.put(key, valuetmp + map.get(key)); logger.debug("?UsageRecord -designDocId:{}", designDocId); keyOrderingProperty = key; @@ -1498,35 +1495,35 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc } } } - + // convert usage from byte to Mb - if (totalFilters instanceof UsageStorageValue) { + if(totalFilters instanceof UsageStorageValue) { totalQuota = totalQuota / 1024 / 1024; totalQuota = Math.round(totalQuota * 100.0) / 100.0; } totalFilters.setOrderingProperty(keyOrderingProperty); - if (totalQuota.isNaN()) { + if(totalQuota.isNaN()) { totalQuota = 0.0; } totalFilters.setD(totalQuota); } - + return listUsage; } - + @Override public String getRecord(String recordId, String recordType) throws Exception { - + try { Bucket bucket = getBucket(recordType); JsonDocument recordJson = bucket.get(recordId); return recordJson.content().toString(); - } catch (Exception e) { + } catch(Exception e) { return null; } - + } - + /** * Used for storage status aka tab Space (into portlet accounting) and * popolate a list combobox used @@ -1535,43 +1532,43 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc */ @Override public SortedSet getSpaceProvidersIds() throws Exception { - String currentScope = ScopeProvider.instance.get(); - + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); + JsonArray startKey = JsonArray.create(); startKey.add(currentScope); - + int scopeDateGroupLevel = 2; int groupLevel = scopeDateGroupLevel; String designDocId = "StorageStatusRecordValue"; String viewName = "providerId"; logger.trace("designDocId:{} view:{} startKey:{} groupLevel:{}", designDocId, viewName, startKey, groupLevel); ViewQuery query = ViewQuery.from(designDocId, viewName); - + query.inclusiveEnd(); query.groupLevel(groupLevel); query.startKey(startKey); query.descending(false); SortedSet ret = new TreeSet(); - + Bucket bucket = getBucket(AggregatedStorageStatusRecord.class); - + ViewResult viewResult; try { // execute query in a specify bucket viewResult = bucket.query(query); - - } catch (Exception e) { + + } catch(Exception e) { logger.error("error executing the query", e); throw e; } - for (ViewRow row : viewResult) { + for(ViewRow row : viewResult) { String value = (String) row.value(); ret.add(value); - + } return ret; } - + /** * used for accounting portlet section storage status * @@ -1582,37 +1579,36 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc * @return SortedMap * @throws JSONException */ - public SortedMap> getSpaceTimeSeries(Class> clz, + public SortedMap> getSpaceTimeSeries(Class> clz, TemporalConstraint temporalConstraint, List filters, List providersId) throws Exception { - - String currentScope = null; - currentScope = ScopeProvider.instance.get(); - + + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); + JsonArray startKey = JsonArray.create(); - + JsonArray endKey = JsonArray.create(); startKey.add(currentScope); endKey.add(currentScope); - + AggregationMode aggregationMode = temporalConstraint.getAggregationMode(); - + JsonArray temporalStartKey = getRangeKey(temporalConstraint.getStartTime(), aggregationMode, false, false); - + JsonArray temporalEndKey = getRangeKey(temporalConstraint.getEndTime(), aggregationMode, false, false); Collection keys = new TreeSet<>(); - + String designDocId = "StorageStatusUsage"; String viewName; int groupLevel = 5; - if (temporalConstraint.getAggregationMode().equals(AggregationMode.MONTHLY)) { + if(temporalConstraint.getAggregationMode().equals(AggregationMode.MONTHLY)) { groupLevel = 4; } - if (temporalConstraint.getAggregationMode().equals(AggregationMode.YEARLY)) { + if(temporalConstraint.getAggregationMode().equals(AggregationMode.YEARLY)) { groupLevel = 3; } viewName = temporalConstraint.getAggregationMode().name().toLowerCase(); - - if (filters != null && filters.size() != 0) { + + if(filters != null && filters.size() != 0) { // Sorting filter for call a mapreduce Collections.sort(filters, new Comparator() { @Override @@ -1621,118 +1617,118 @@ public class AccountingPersistenceQueryCouchBase implements AccountingPersistenc return result; } }); - for (Filter filter : filters) { + for(Filter filter : filters) { String filterKey = filter.getKey(); String filterValue = filter.getValue(); - - if (filterKey != null && filterKey.compareTo("") != 0) { - if (filterValue != null && filterValue.compareTo("") != 0) { - if (keys.contains(filterKey)) { + + if(filterKey != null && filterKey.compareTo("") != 0) { + if(filterValue != null && filterValue.compareTo("") != 0) { + if(keys.contains(filterKey)) { throw new DuplicatedKeyFilterException("Only one value per Filter key is allowed"); } startKey.add(filterValue); endKey.add(filterValue); keys.add(filterKey); viewName = viewName + "_" + filterKey; - if (filterKey != "consumerId") { + if(filterKey != "consumerId") { groupLevel = groupLevel + 1; } } else { throw new KeyException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } - + } else { throw new ValueException( String.format("Invalid %s : %s", Filter.class.getSimpleName(), filter.toString())); } } } - for (Object temporal : temporalStartKey.toList()) { - if (!temporal.toString().isEmpty()) + for(Object temporal : temporalStartKey.toList()) { + if(!temporal.toString().isEmpty()) startKey.add(temporal); } int count = 1; - for (Object temporal : temporalEndKey.toList()) { - if (!temporal.toString().isEmpty()) { + for(Object temporal : temporalEndKey.toList()) { + if(!temporal.toString().isEmpty()) { // couchbase exclude last value - if (count == temporalEndKey.size()) + if(count == temporalEndKey.size()) temporal = (int) temporal + 1; endKey.add(temporal); } count++; } - + ViewQuery query = ViewQuery.from(designDocId, viewName); query.inclusiveEnd(); query.groupLevel(groupLevel); query.startKey(startKey); query.endKey(endKey); query.descending(false); - + logger.trace( "Bucket :{}, Design Doc ID : {}, View Name : {}, " + "Group Level : {}, Start Key : {}, End Key : {}," + "temporalStartKey :{}, temporalEndKey :{}", clz.getSimpleName(), designDocId, viewName, groupLevel, startKey, endKey, temporalStartKey.toString(), temporalEndKey.toString()); - + @SuppressWarnings("unchecked") Bucket bucket = getBucket((Class) clz); - + ViewResult viewResult; try { // execute query in a specify bucket viewResult = bucket.query(query); - - } catch (Exception e) { + + } catch(Exception e) { logger.error(e.getLocalizedMessage()); throw e; } - - SortedMap> ret = new TreeMap<>(); - - for (ViewRow row : viewResult) { - + + SortedMap> ret = new TreeMap<>(); + + for(ViewRow row : viewResult) { + JsonArray array = (JsonArray) row.key(); Calendar calendar = getCalendarFromArray(array); - + JsonObject value = (JsonObject) row.value(); JSONObject obj = new JSONObject(value.toString()); - + // logger.trace("row: {}, value: {}, obj: // {}",row.toString(),value.toString(),obj.toString()); - for (Iterator iterator = obj.keys(); iterator.hasNext();) { + for(Iterator iterator = obj.keys(); iterator.hasNext();) { String key = (String) iterator.next(); String[] tmp = key.split("-"); String providerId = tmp[0]; - - if (providersId.contains(providerId)) { + + if(providersId.contains(providerId)) { Long valueProvider = Long.parseLong(obj.get(key).toString().split(",")[0].replace("[", "")); // convert into kb valueProvider = valueProvider / 1024; - + Filter filter = new Filter("providerId", providerId); - if (!ret.containsKey(filter)) { - SortedMap infos = new TreeMap<>(); + if(!ret.containsKey(filter)) { + SortedMap infos = new TreeMap<>(); infos.put(calendar, valueProvider); ret.put(filter, infos); } else { - SortedMap singleValue = ret.get(filter); - if (!singleValue.containsKey(calendar)) { + SortedMap singleValue = ret.get(filter); + if(!singleValue.containsKey(calendar)) { // if not exist put singleValue.put(calendar, valueProvider); } else { // if exist, update singleValue.put(calendar, singleValue.get(calendar) + valueProvider); } - + } } - + } } logger.trace("return ret:{}", ret.toString()); return ret; } - + } diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java index e3899dc..92a846d 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/AccountingPersistenceQueryCouchBaseTest.java @@ -1,8 +1,4 @@ package org.gcube.accounting.analytics.persistence.couchbase; -/** - * - */ - import java.util.ArrayList; import java.util.Calendar; @@ -40,61 +36,56 @@ import org.slf4j.LoggerFactory; * */ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { - + private static Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryCouchBaseTest.class); - + protected AccountingPersistenceQueryCouchBase accountingPersistenceQueryCouchBase; - + public class ExtendedInfo extends Info { - + protected String key; - + /** * @return the key */ public String getKey() { return key; } - + /** * @param key the key to set */ public void setKey(String key) { this.key = key; } - - public ExtendedInfo(String key, Calendar calendar, JSONObject value){ + + public ExtendedInfo(String key, Calendar calendar, JSONObject value) { super(calendar, value); this.key = key; } - - public String toString(){ - String info = String .format("Key : %s, %s ", key, super.toString()); + + public String toString() { + String info = String.format("Key : %s, %s ", key, super.toString()); return info; } - + } - - - - + @Before - public void before() throws Exception{ - AccountingPersistenceBackendQueryConfiguration configuration = new - AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchBase.class); - + public void before() throws Exception { + AccountingPersistenceBackendQueryConfiguration configuration = new AccountingPersistenceBackendQueryConfiguration( + AccountingPersistenceQueryCouchBase.class); + accountingPersistenceQueryCouchBase = new AccountingPersistenceQueryCouchBase(); accountingPersistenceQueryCouchBase.prepareConnection(configuration); } - - - public void printMap(Map map){ - for(Info info : map.values()){ + public void printMap(Map map) { + for(Info info : map.values()) { logger.debug("{}", info); } } - + @Test public void getUsersInVREs() throws Exception { List contexts = new ArrayList<>(); @@ -112,25 +103,22 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { endTime.set(2017, Calendar.JULY, 15); logger.debug("EndTime {}", endTime.getTimeInMillis()); - - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.MONTHLY); - - Class clz = - AggregatedServiceUsageRecord.class; + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.MONTHLY); + + Class clz = AggregatedServiceUsageRecord.class; List filters = new ArrayList(); //Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "DataMiner"); Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "RConnector"); filters.add(filter); - - for(String context : contexts){ + for(String context : contexts) { ScopeProvider.instance.set(context); - SortedSet top = accountingPersistenceQueryCouchBase.getNextPossibleValuesWithMap(clz, temporalConstraint, filters, ServiceUsageRecord.CONSUMER_ID, null); + SortedSet top = accountingPersistenceQueryCouchBase.getNextPossibleValuesWithMap(clz, + temporalConstraint, filters, ServiceUsageRecord.CONSUMER_ID, null); logger.info("Context : {} - Users [{}] : {}", context, top.size(), top); @@ -149,102 +137,82 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { endTime.set(2017, Calendar.JULY, 15); logger.debug("EndTime {}", endTime.getTimeInMillis()); - List filters = new ArrayList(); Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "DataMiner"); //Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "RConnector"); filters.add(filter); + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.MONTHLY); + + Class clz = AggregatedServiceUsageRecord.class; + + SortedMap> set = accountingPersistenceQueryCouchBase.getTopValues(clz, + temporalConstraint, filters, ServiceUsageRecord.CONSUMER_ID, null); - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.MONTHLY); - - Class clz = - AggregatedServiceUsageRecord.class; - - SortedMap> set = - accountingPersistenceQueryCouchBase.getTopValues( - clz, temporalConstraint, filters, - ServiceUsageRecord.CONSUMER_ID, null); - - logger.debug("Result {}", set); } - + @Test - public void testTimeSeries() throws Exception{ - + public void testTimeSeries() throws Exception { + + //AccountingPersistenceQueryFactory.getForcedScope().set("/gcube/devNext/NextNext"); + Calendar startTime = Calendar.getInstance(); - startTime.set(2015, Calendar.AUGUST, 20); + startTime.set(2017, Calendar.NOVEMBER, 29); Calendar endTime = Calendar.getInstance(); - endTime.set(2016, Calendar.SEPTEMBER, 29,23,59); + endTime.set(2017, Calendar.DECEMBER, 29); List filters = new ArrayList(); - - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); - Class clz = - AggregatedJobUsageRecord.class; - SortedMap set = - accountingPersistenceQueryCouchBase.getTimeSeries( - clz, temporalConstraint, filters); + + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + Class clz = AggregatedJobUsageRecord.class; + SortedMap set = accountingPersistenceQueryCouchBase.getTimeSeries(clz, temporalConstraint, + filters); logger.debug("Result final{}", set); } - + @Test - public void testTimeSeriesNoContext() throws Exception{ - + public void testTimeSeriesNoContext() throws Exception { + Calendar startTime = Calendar.getInstance(); startTime.set(2017, Calendar.APRIL, 20); Calendar endTime = Calendar.getInstance(); - endTime.set(2017, Calendar.APRIL, 29,23,59); + endTime.set(2017, Calendar.APRIL, 29, 23, 59); List filters = new ArrayList(); - + /* Filter filter = new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService"); filters.add(filter); */ - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); - Class clz = - AggregatedServiceUsageRecord.class; - SortedMap set = - accountingPersistenceQueryCouchBase.getNoContextTimeSeries( - clz, temporalConstraint, filters); + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + Class clz = AggregatedServiceUsageRecord.class; + SortedMap set = accountingPersistenceQueryCouchBase.getNoContextTimeSeries(clz, + temporalConstraint, filters); logger.debug("Result final{}", set); } - - - + @Test - public void getUsageValue() throws Exception{ + public void getUsageValue() throws Exception { Calendar startTime = Calendar.getInstance(); startTime.set(2015, Calendar.MAY, 1); Calendar endTime = Calendar.getInstance(); ScopeProvider.instance.set("/gcube"); - Filter filter = - new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"); - - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); - - JSONObject filterValue = - accountingPersistenceQueryCouchBase.getUsageValue(AggregatedServiceUsageRecord.class, - temporalConstraint, filter); - - logger.info("result:"+filterValue.toString()); - + Filter filter = new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"); + + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + + JSONObject filterValue = accountingPersistenceQueryCouchBase.getUsageValue(AggregatedServiceUsageRecord.class, + temporalConstraint, filter); + + logger.info("result:" + filterValue.toString()); + } - - - - /** * utilizzato per effettuare una singola chiamata ad accounting analytics in modo che ritorni una lista di quote riempite * @@ -278,44 +246,39 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { * ] */ @Test - public void getUsageValueQuotaTotal() throws Exception{ + public void getUsageValueQuotaTotal() throws Exception { String context = ScopedTest.getCurrentScope(DEFAULT_TEST_SCOPE); Calendar startTime = Calendar.getInstance(); startTime.set(2017, Calendar.MAY, 1); Calendar endTime = Calendar.getInstance(); - - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); - + + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); /*ask quota for user alessandro pieve * * SERVICE * */ - List filters=new ArrayList(); + List filters = new ArrayList(); filters.add(new Filter("serviceClass", "DataAccess")); filters.add(new Filter("serviceName", "CkanConnector")); - UsageValue totalfilter=new UsageServiceValue(context,"lucio.lelii",AggregatedServiceUsageRecord.class,temporalConstraint,filters); - - - - + UsageValue totalfilter = new UsageServiceValue(context, "lucio.lelii", AggregatedServiceUsageRecord.class, + temporalConstraint, filters); /**** *Example call storage status for each consumer id (quota total used ) */ //Call quota total for consumerID - UsageValue totalfilterStorageStatus=new UsageStorageValue(context,"name.surname",AggregatedStorageStatusRecord.class); - UsageValue totalfilterStorageStatus_2=new UsageStorageValue(context,"lucio.lelii",AggregatedStorageStatusRecord.class); - UsageValue totalfilterStorageStatus_3=new UsageStorageValue(context,"alessandro.pieve",AggregatedStorageStatusRecord.class); - UsageValue totalfilterStorageStatus_4=new UsageStorageValue(context,"giancarlo.panichi",AggregatedStorageStatusRecord.class); - - - - + UsageValue totalfilterStorageStatus = new UsageStorageValue(context, "name.surname", + AggregatedStorageStatusRecord.class); + UsageValue totalfilterStorageStatus_2 = new UsageStorageValue(context, "lucio.lelii", + AggregatedStorageStatusRecord.class); + UsageValue totalfilterStorageStatus_3 = new UsageStorageValue(context, "alessandro.pieve", + AggregatedStorageStatusRecord.class); + UsageValue totalfilterStorageStatus_4 = new UsageStorageValue(context, "giancarlo.panichi", + AggregatedStorageStatusRecord.class); /**** *Example call storage status for each consumer id( quota into period) @@ -327,14 +290,12 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { endTimeStorage.set(2017, Calendar.APRIL, 13); /*TemporalConstraint temporalConstraintStorage =new TemporalConstraint(startTimeStorage.getTimeInMillis(), endTimeStorage.getTimeInMillis(), AggregationMode.DAILY);*/ - TemporalConstraint temporalConstraintStorage =null; - UsageValue totalfilterStorageStatusPeriod=new UsageStorageValue(context,"alessandro.pieve",AggregatedStorageStatusRecord.class,temporalConstraintStorage); - - - + TemporalConstraint temporalConstraintStorage = null; + UsageValue totalfilterStorageStatusPeriod = new UsageStorageValue(context, "alessandro.pieve", + AggregatedStorageStatusRecord.class, temporalConstraintStorage); //richiedo la lista di dati usati totali - List listTotalFilter=new ArrayList(); + List listTotalFilter = new ArrayList(); listTotalFilter.add(totalfilter); listTotalFilter.add(totalfilterStorageStatus); listTotalFilter.add(totalfilterStorageStatus_2); @@ -342,54 +303,44 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { listTotalFilter.add(totalfilterStorageStatus_4); listTotalFilter.add(totalfilterStorageStatusPeriod); - - logger.info("filterPackageQuota:"+listTotalFilter); - List filterValue = - accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(listTotalFilter); - logger.info("result:"+filterValue.toString()); + logger.info("filterPackageQuota:" + listTotalFilter); + List filterValue = accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(listTotalFilter); + logger.info("result:" + filterValue.toString()); } - - - - - @Test - public void getQuerableKeyJob() throws Exception{ + @Test + public void getQuerableKeyJob() throws Exception { SortedSet keys; - + keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedJobUsageRecord.class); - for (String key : keys) { - if (key != null && !key.isEmpty()) { - logger.debug("key:" +key); + for(String key : keys) { + if(key != null && !key.isEmpty()) { + logger.debug("key:" + key); } } logger.debug("List FilterKeys:" + keys.toString()); } - + @Test public void testTopService() throws Exception { Calendar startTime = Calendar.getInstance(); startTime.set(2016, Calendar.AUGUST, 27); Calendar endTime = Calendar.getInstance(); - endTime.set(2016, Calendar.SEPTEMBER, 28,23,59); - + endTime.set(2016, Calendar.SEPTEMBER, 28, 23, 59); + List filters = new ArrayList(); //filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_NAME, "IS-Registry")); //filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_CLASS, "Common")); - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); - - Class clz = - AggregatedServiceUsageRecord.class; - - SortedMap> set = - accountingPersistenceQueryCouchBase.getTopValues( - clz, temporalConstraint, filters, - AggregatedServiceUsageRecord.OPERATION_RESULT, null); - + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + + Class clz = AggregatedServiceUsageRecord.class; + + SortedMap> set = accountingPersistenceQueryCouchBase.getTopValues(clz, + temporalConstraint, filters, AggregatedServiceUsageRecord.OPERATION_RESULT, null); + logger.debug("Result final{}", set); - + } @Test @@ -397,298 +348,250 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { Calendar startTime = Calendar.getInstance(); startTime.set(2017, Calendar.FEBRUARY, 1); Calendar endTime = Calendar.getInstance(); - endTime.set(2017, Calendar.FEBRUARY, 28,23,59); - - + endTime.set(2017, Calendar.FEBRUARY, 28, 23, 59); + List filters = new ArrayList(); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "scarponi")); - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); - - Class clz = - AggregatedStorageUsageRecord.class; - - SortedMap> set = - accountingPersistenceQueryCouchBase.getTopValues( - clz, temporalConstraint, filters, - AggregatedStorageUsageRecord.OPERATION_TYPE, null); - + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + + Class clz = AggregatedStorageUsageRecord.class; + + SortedMap> set = accountingPersistenceQueryCouchBase.getTopValues(clz, + temporalConstraint, filters, AggregatedStorageUsageRecord.OPERATION_TYPE, null); + logger.debug("Result final{}", set); - + } - - - - - - - - - - @Test - public void getQuerableKeyService() throws Exception{ + @Test + public void getQuerableKeyService() throws Exception { SortedSet keys; - + keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedServiceUsageRecord.class); - for (String key : keys) { - if (key != null && !key.isEmpty()) { - logger.debug("key:" +key); + for(String key : keys) { + if(key != null && !key.isEmpty()) { + logger.debug("key:" + key); } } logger.debug("List FilterKeys:" + keys.toString()); } - - public static SortedMap padMap( - SortedMap unpaddedData, + + public static SortedMap padMap(SortedMap unpaddedData, TemporalConstraint temporalConstraint) throws Exception { JSONObject jsonObject = getPaddingJSONObject(unpaddedData); SortedSet sequence = temporalConstraint.getCalendarSequence(); - for (Calendar progressTime : sequence) { + for(Calendar progressTime : sequence) { Info info = unpaddedData.get(progressTime); - if (info == null) { + if(info == null) { info = new Info(progressTime, jsonObject); unpaddedData.put(progressTime, info); } } return unpaddedData; } - public static JSONObject getPaddingJSONObject( - Map unpaddedResults) throws JSONException { - + + public static JSONObject getPaddingJSONObject(Map unpaddedResults) throws JSONException { + JSONObject jsonObject = new JSONObject(); //verify data consistency - - if (unpaddedResults.size()!=0){ + + if(unpaddedResults.size() != 0) { Info auxInfo = new ArrayList(unpaddedResults.values()).get(0); JSONObject auxJsonObject = auxInfo.getValue(); @SuppressWarnings("unchecked") Iterator keys = auxJsonObject.keys(); - while (keys.hasNext()) { + while(keys.hasNext()) { String key = keys.next(); jsonObject.put(key, 0); } } - + return jsonObject; } - + @Test public void testFull() throws Exception { logger.debug("test full "); - try{ + try { Calendar startTime = Calendar.getInstance(); //startTime.set(2016, Calendar.AUGUST, 20, 00, 00); - startTime.set(2016, Calendar.AUGUST, 29,00,00); + startTime.set(2016, Calendar.AUGUST, 29, 00, 00); Calendar endTime = Calendar.getInstance(); //endTime.set(2016, Calendar.AUGUST, 29, 23, 59); - endTime.set(2016, Calendar.AUGUST, 31,23,59); - - + endTime.set(2016, Calendar.AUGUST, 31, 23, 59); + List filters = new ArrayList(); - - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); - - Class clz = - AggregatedServiceUsageRecord.class; - - SortedMap> set = - accountingPersistenceQueryCouchBase.getTopValues( - clz, temporalConstraint, filters, - AggregatedServiceUsageRecord.CALLED_METHOD, null); + + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + + Class clz = AggregatedServiceUsageRecord.class; + + SortedMap> set = accountingPersistenceQueryCouchBase + .getTopValues(clz, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, null); /**pad*/ - int limit=0; - boolean pad=true; + int limit = 0; + boolean pad = true; int count = set.size() > limit ? limit : set.size(); NumberedFilter firstRemovalKey = null; - logger.debug("set completo"+set.toString()); - for(NumberedFilter nf : set.keySet()){ - if(--count>=0 || limit<=0){ - if(pad){ + logger.debug("set completo" + set.toString()); + for(NumberedFilter nf : set.keySet()) { + if(--count >= 0 || limit <= 0) { + if(pad) { padMap(set.get(nf), temporalConstraint); } - }else{ - if(firstRemovalKey==null){ + } else { + if(firstRemovalKey == null) { firstRemovalKey = nf; - }else{ + } else { break; } } } - if(firstRemovalKey!=null){ - logger.debug("First removal key set:"+set.subMap(set.firstKey(), firstRemovalKey)); + if(firstRemovalKey != null) { + logger.debug("First removal key set:" + set.subMap(set.firstKey(), firstRemovalKey)); } - - - logger.debug("set: "+set); - - - }catch(Exception e){ + + logger.debug("set: " + set); + + } catch(Exception e) { e.printStackTrace(); } } - @Test public void testContextService() throws Exception { Calendar startTime = Calendar.getInstance(); startTime.set(2017, Calendar.APRIL, 20); Calendar endTime = Calendar.getInstance(); - endTime.set(2017, Calendar.APRIL, 28,23,59); + endTime.set(2017, Calendar.APRIL, 28, 23, 59); List filters = new ArrayList(); //filters.add(new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService")); //filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); - - Class clz = - AggregatedServiceUsageRecord.class; - - List context=new ArrayList(); + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.DAILY); + Class clz = AggregatedServiceUsageRecord.class; + + List context = new ArrayList(); // , /gcube/devsec/TestCreation12, /gcube/devsec/TestCreation13, /gcube/devsec/TestCreation14, /gcube/devsec/TestCreation15, /gcube/devsec/TestCreation16, /gcube/devsec/TestCreation17, /gcube/devsec/TestCreation7, /gcube/devsec/TestCreation8, /gcube/devsec/TestCreation9, /gcube/devsec/TestLucio2]] context.add("/gcube"); context.add("/gcube/devNext"); - /* - context.add("/gcube/devNext/Luciotest"); - context.add("/gcube/devNext/NextNext"); - context.add("/gcube/devsec"); - context.add("/gcube/devsec/SmartCamera"); - context.add("/gcube/devsec/statVRE"); - context.add("/gcube/devsec/TestAddLast"); - - context.add("/gcube/devsec/devVRE"); - context.add("/gcube/devsec/preVRE"); - context.add("/gcube/preprod/preVRE"); - context.add("/gcube/preprod"); - */ - SortedMap> setContext = - accountingPersistenceQueryCouchBase.getContextTimeSeries( - clz, temporalConstraint, filters, - context); + /* + context.add("/gcube/devNext/Luciotest"); + context.add("/gcube/devNext/NextNext"); + context.add("/gcube/devsec"); + context.add("/gcube/devsec/SmartCamera"); + context.add("/gcube/devsec/statVRE"); + context.add("/gcube/devsec/TestAddLast"); + + context.add("/gcube/devsec/devVRE"); + context.add("/gcube/devsec/preVRE"); + context.add("/gcube/preprod/preVRE"); + context.add("/gcube/preprod"); + */ + SortedMap> setContext = accountingPersistenceQueryCouchBase + .getContextTimeSeries(clz, temporalConstraint, filters, context); logger.debug("Result final{}", setContext); - + } - @Test - public void getListUsage() throws Exception{ + public void getListUsage() throws Exception { Calendar startTime = Calendar.getInstance(); startTime.set(2015, Calendar.SEPTEMBER, 1); Calendar endTime = Calendar.getInstance(); - endTime.set(2016, Calendar.OCTOBER, 20,23,59); + endTime.set(2016, Calendar.OCTOBER, 20, 23, 59); List filters = new ArrayList(); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "valentina.marioli")); ////TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.DAILY); + new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY); //String context="/gcube/devNext"; - List parameters=new ArrayList(); + List parameters = new ArrayList(); parameters.add("serviceClass"); parameters.add("serviceName"); //Class clz = - // AggregatedServiceUsageRecord.class; - // SortedMap result= accountingPersistenceQueryCouchBase.getListUsage(clz,temporalConstraint, filters,context,parameters); - + // AggregatedServiceUsageRecord.class; + // SortedMap result= accountingPersistenceQueryCouchBase.getListUsage(clz,temporalConstraint, filters,context,parameters); + } - - @Test - public void getRecord() throws Exception{ - String recordId="91e1c339-d811-45d7-a13f-7385af59e3c8"; - String type="service"; + public void getRecord() throws Exception { + String recordId = "91e1c339-d811-45d7-a13f-7385af59e3c8"; + String type = "service"; + + String document = accountingPersistenceQueryCouchBase.getRecord(recordId, type); + logger.debug("document:{}", document); - String document=accountingPersistenceQueryCouchBase.getRecord(recordId, type); - logger.debug("document:{}",document); - - } - - @Test - public void getQuerableKeyStorageStatus() throws Exception{ + @Test + public void getQuerableKeyStorageStatus() throws Exception { SortedSet keys; keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedStorageStatusRecord.class); - for (String key : keys) { - if (key != null && !key.isEmpty()) { - logger.debug("key:" +key); + for(String key : keys) { + if(key != null && !key.isEmpty()) { + logger.debug("key:" + key); } } logger.debug("List FilterKeys:" + keys.toString()); } @Test - public void testGetSpaceProvidersIds() throws Exception{ - SortedSet listProvidersId = - accountingPersistenceQueryCouchBase.getSpaceProvidersIds(); + public void testGetSpaceProvidersIds() throws Exception { + SortedSet listProvidersId = accountingPersistenceQueryCouchBase.getSpaceProvidersIds(); logger.debug("Result final{}", listProvidersId); } - @Test - public void testGetFilterValue() throws Exception{ + public void testGetFilterValue() throws Exception { //String key="consumerId"; - String key="dataServiceId"; + String key = "dataServiceId"; - SortedSet listFilterValue = - accountingPersistenceQueryCouchBase.getFilterValues(AggregatedStorageStatusRecord.class, null, null, key, null); + SortedSet listFilterValue = accountingPersistenceQueryCouchBase + .getFilterValues(AggregatedStorageStatusRecord.class, null, null, key, null); logger.debug("Result final{}", listFilterValue); } - @Test public void testUsageStorage() throws Exception { Calendar startTime = Calendar.getInstance(); startTime.set(2017, Calendar.APRIL, 13); Calendar endTime = Calendar.getInstance(); - endTime.set(2017, Calendar.APRIL, 26,23,59); + endTime.set(2017, Calendar.APRIL, 26, 23, 59); List filters = new ArrayList(); //filters.add(new Filter(AggregatedStorageStatusRecord.DATA_SERVICEID, "identifier")); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); - TemporalConstraint temporalConstraint = - new TemporalConstraint(startTime.getTimeInMillis(), - endTime.getTimeInMillis(), AggregationMode.YEARLY); - - - List providerId=new ArrayList(); - Class clz = - AggregatedStorageStatusRecord.class; + TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), + endTime.getTimeInMillis(), AggregationMode.YEARLY); + + List providerId = new ArrayList(); + Class clz = AggregatedStorageStatusRecord.class; //providerId.add("Rstudio"); providerId.add("MongoDb"); - - SortedMap> setContext = - accountingPersistenceQueryCouchBase.getSpaceTimeSeries( - clz, - temporalConstraint, filters, - providerId); + SortedMap> setContext = accountingPersistenceQueryCouchBase + .getSpaceTimeSeries(clz, temporalConstraint, filters, providerId); - - - - int count = setContext.size(); Filter firstRemovalKey = null; - for(Filter nf : setContext.keySet()){ - if(--count>=0){ + for(Filter nf : setContext.keySet()) { + if(--count >= 0) { //if(pad){ padMapStorage(setContext.get(nf), temporalConstraint); - + //} - }else{ - if(firstRemovalKey==null){ + } else { + if(firstRemovalKey == null) { firstRemovalKey = nf; - }else{ + } else { break; } } @@ -700,32 +603,27 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { } */ - - logger.debug("Result final{}", setContext); - + } - public SortedMap padMapStorage( - SortedMap unpaddedData, + public SortedMap padMapStorage(SortedMap unpaddedData, TemporalConstraint temporalConstraint) throws Exception { - + //JSONObject jsonObject = getPaddingJSONObject(unpaddedData); SortedSet sequence = temporalConstraint.getCalendarSequence(); Long longValuePre = null; - for (Calendar progressTime : sequence) { + for(Calendar progressTime : sequence) { Long longValue = unpaddedData.get(progressTime); - if (longValue == null) { + if(longValue == null) { unpaddedData.put(progressTime, longValuePre); - }else{ - longValuePre=longValue; + } else { + longValuePre = longValue; } } return unpaddedData; } - - } diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/ScopedTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/ScopedTest.java index 2f625d4..9eac682 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/ScopedTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/couchbase/ScopedTest.java @@ -1,6 +1,3 @@ -/** - * - */ package org.gcube.accounting.analytics.persistence.couchbase; import java.io.IOException; @@ -23,12 +20,9 @@ import org.slf4j.LoggerFactory; */ public class ScopedTest { - - - private static final Logger logger = LoggerFactory.getLogger(ScopedTest.class); - protected static final String PROPERTIES_FILENAME = "token.properties"; + protected static final String PROPERTIES_FILENAME = "token.properties"; private static final String GCUBE_DEVNEXT_VARNAME = "GCUBE_DEVNEXT"; public static final String GCUBE_DEVNEXT; @@ -45,21 +39,20 @@ public class ScopedTest { public static final String SA_VARNAME = "SA"; public static final String SA; - public static final String DEFAULT_TEST_SCOPE; public static final String ALTERNATIVE_TEST_SCOPE; static { Properties properties = new Properties(); InputStream input = ScopedTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME); - + try { // load the properties file properties.load(input); - } catch (IOException e) { + } catch(IOException e) { throw new RuntimeException(e); } - + GCUBE_DEVNEXT = properties.getProperty(GCUBE_DEVNEXT_VARNAME); GCUBE_DEVNEXT_NEXTNEXT = properties.getProperty(GCUBE_DEVNEXT_NEXTNEXT_VARNAME); @@ -68,30 +61,29 @@ public class ScopedTest { SA = properties.getProperty(SA_VARNAME); - DEFAULT_TEST_SCOPE = SA; + DEFAULT_TEST_SCOPE = GCUBE_DEVNEXT; ALTERNATIVE_TEST_SCOPE = GCUBE_DEVSEC; } - public static String getCurrentScope(String token) throws ObjectNotFound, Exception{ + public static String getCurrentScope(String token) throws ObjectNotFound, Exception { AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); String context = authorizationEntry.getContext(); logger.info("Context of token {} is {}", token, context); return context; } - - public static void setContext(String token) throws ObjectNotFound, Exception{ + public static void setContext(String token) throws ObjectNotFound, Exception { SecurityTokenProvider.instance.set(token); ScopeProvider.instance.set(getCurrentScope(token)); } @BeforeClass - public static void beforeClass() throws Exception{ + public static void beforeClass() throws Exception { setContext(DEFAULT_TEST_SCOPE); } @AfterClass - public static void afterClass() throws Exception{ + public static void afterClass() throws Exception { SecurityTokenProvider.instance.reset(); ScopeProvider.instance.reset(); }