From 1e78ae0fcd8c083774312e3b8d5a07c151250c88 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 21 Jul 2015 14:33:36 +0000 Subject: [PATCH] Implemented padded results and related functionality git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics@117390 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 6 + .../accounting/analytics/BasicQuery.java | 42 ------ .../org/gcube/accounting/analytics/Info.java | 2 +- .../analytics/ResourceRecordQuery.java | 70 +++++++++- .../analytics/TemporalConstraint.java | 120 +++++++++++++++++- .../AccountingPersistenceQuery.java | 6 +- .../analytics/TemporalConstraintTest.java | 109 ++++++++++++++++ 7 files changed, 306 insertions(+), 49 deletions(-) delete mode 100644 src/main/java/org/gcube/accounting/analytics/BasicQuery.java create mode 100644 src/test/java/org/gcube/accounting/analytics/TemporalConstraintTest.java diff --git a/pom.xml b/pom.xml index 714a0d3..759bc79 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,12 @@ 4.11 test + + ch.qos.logback + logback-classic + 1.0.13 + test + \ No newline at end of file diff --git a/src/main/java/org/gcube/accounting/analytics/BasicQuery.java b/src/main/java/org/gcube/accounting/analytics/BasicQuery.java deleted file mode 100644 index 8a98ba6..0000000 --- a/src/main/java/org/gcube/accounting/analytics/BasicQuery.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * - */ -package org.gcube.accounting.analytics; - -import java.util.List; - -import org.gcube.accounting.analytics.exception.NoAvailableScopeException; -import org.gcube.accounting.analytics.exception.NoUsableAccountingPersistenceQueryFound; -import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; -import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory; -import org.gcube.common.scope.api.ScopeProvider; - -/** - * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ - */ -public class BasicQuery { - - protected AccountingPersistenceQuery accountingPersistenceQuery; - - public BasicQuery() throws NoAvailableScopeException, NoUsableAccountingPersistenceQueryFound { - this.accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance(); - } - - public BasicQuery(String scope) throws NoAvailableScopeException, NoUsableAccountingPersistenceQueryFound { - ScopeProvider.instance.set(scope); - this.accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance(); - } - - public List getServiceInfo(TemporalConstraint temporalConstraint) { - - - return null; - } - - public List getStorageInfo(TemporalConstraint temporalConstraint) { - - - return null; - } - -} diff --git a/src/main/java/org/gcube/accounting/analytics/Info.java b/src/main/java/org/gcube/accounting/analytics/Info.java index 5a27eee..4ec5bdd 100644 --- a/src/main/java/org/gcube/accounting/analytics/Info.java +++ b/src/main/java/org/gcube/accounting/analytics/Info.java @@ -60,7 +60,7 @@ public class Info { @Override public String toString(){ SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SS z"); - simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + simpleDateFormat.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); return String.format("Date : %s, Value : %s", simpleDateFormat.format(date), value.toString()); } } diff --git a/src/main/java/org/gcube/accounting/analytics/ResourceRecordQuery.java b/src/main/java/org/gcube/accounting/analytics/ResourceRecordQuery.java index 9b91d76..7fc4046 100644 --- a/src/main/java/org/gcube/accounting/analytics/ResourceRecordQuery.java +++ b/src/main/java/org/gcube/accounting/analytics/ResourceRecordQuery.java @@ -3,7 +3,11 @@ */ package org.gcube.accounting.analytics; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -15,6 +19,8 @@ import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFact import org.gcube.accounting.datamodel.SingleUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.common.scope.api.ScopeProvider; +import org.json.JSONException; +import org.json.JSONObject; import org.reflections.Reflections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,14 +65,72 @@ public class ResourceRecordQuery { this.accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance(); } - public List getInfo(Class usageRecordType, - TemporalConstraint temporalConstraint, List filters) throws Exception{ - return accountingPersistenceQuery.query(usageRecordType, temporalConstraint, filters); + protected static JSONObject getPaddingJSONObject(Map unpaddedResults) throws JSONException{ + Info auxInfo = new ArrayList(unpaddedResults.values()).get(0); + JSONObject auxJsonObject = auxInfo.getValue(); + @SuppressWarnings("unchecked") + Iterator keys = auxJsonObject.keys(); + + JSONObject jsonObject = new JSONObject(); + while(keys.hasNext()){ + String key = keys.next(); + jsonObject.put(key, 0); + } + + return jsonObject; } + public static List getPaddedResults(Map unpaddedResults, TemporalConstraint temporalConstraint) throws JSONException{ + JSONObject jsonObject = getPaddingJSONObject(unpaddedResults); + + List paddedResults = new ArrayList(); + List sequence = temporalConstraint.getCalendarSequence(); + + for(Calendar progressTime : sequence){ + if(unpaddedResults.get(progressTime)!=null){ + paddedResults.add(unpaddedResults.get(progressTime)); + }else{ + Date date = new Date(progressTime.getTimeInMillis()); + Info info = new Info(date, jsonObject); + paddedResults.add(info); + } + } + + return paddedResults; + } + /** + * Return results with padding if pad is set to true. + * @param usageRecordType + * @param temporalConstraint + * @param filters + * @param pad + * @return + * @throws Exception + */ + public List getInfo(Class usageRecordType, + TemporalConstraint temporalConstraint, List filters, boolean pad) throws Exception { + Map unpaddedResults = accountingPersistenceQuery.query(usageRecordType, temporalConstraint, filters); + if(!pad){ + return new ArrayList(unpaddedResults.values()); + } + + return getPaddedResults(unpaddedResults, temporalConstraint); + } + /** + * Return unpadded result + * @param usageRecordType + * @param temporalConstraint + * @param filters + * @return + * @throws Exception + */ + public List getInfo(Class usageRecordType, + TemporalConstraint temporalConstraint, List filters) throws Exception{ + return getInfo(usageRecordType, temporalConstraint, filters, false); + } } diff --git a/src/main/java/org/gcube/accounting/analytics/TemporalConstraint.java b/src/main/java/org/gcube/accounting/analytics/TemporalConstraint.java index df333c0..7b54f43 100644 --- a/src/main/java/org/gcube/accounting/analytics/TemporalConstraint.java +++ b/src/main/java/org/gcube/accounting/analytics/TemporalConstraint.java @@ -3,16 +3,52 @@ */ package org.gcube.accounting.analytics; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ * */ public class TemporalConstraint { - + + private static final Logger logger = LoggerFactory.getLogger(TemporalConstraint.class); + + private static final String UTC_TIME_ZONE = "UTC"; + public static final TimeZone DEFAULT_TIME_ZONE = TimeZone.getTimeZone(UTC_TIME_ZONE); + public enum AggregationMode { YEARLY, MONTHLY, DAILY, HOURLY, MINUTELY, SECONDLY, MILLISECONDLY } + public enum CalendarEnum { + YEAR(Calendar.YEAR), + MONTH(Calendar.MONTH), + DAY(Calendar.DAY_OF_MONTH), + HOUR(Calendar.HOUR_OF_DAY), + MINUTE(Calendar.MINUTE), + SECOND(Calendar.SECOND), + MILLISECOND(Calendar.MILLISECOND); + + private final int calendarValue; + + CalendarEnum(int calendarValue){ + this.calendarValue = calendarValue; + } + + public int getCalendarValue(){ + return calendarValue; + } + + }; + protected long startTime; protected long endTime; protected AggregationMode aggregationMode; @@ -64,6 +100,88 @@ public class TemporalConstraint { public void setAggregationMode(AggregationMode aggregationMode) { this.aggregationMode = aggregationMode; } + + public static String timeInMillisToString(long timeInMillis){ + Date date = new Date(timeInMillis); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SS z"); + simpleDateFormat.setTimeZone(DEFAULT_TIME_ZONE); + return String.format("%s (%d millis)", simpleDateFormat.format(date), timeInMillis); + } + public static Calendar getAlignedCalendar(long millis, AggregationMode aggregationMode){ + Calendar alignedCalendar = Calendar.getInstance(); + alignedCalendar.setTimeZone(DEFAULT_TIME_ZONE); + alignedCalendar.setTimeInMillis(millis); + + CalendarEnum[] calendarValues = CalendarEnum.values(); + + for(int i=aggregationMode.ordinal()+1; i getCalendarSequence(){ + List sequence = new ArrayList(); + + CalendarEnum[] calendarValues = CalendarEnum.values(); + int calendarValue = calendarValues[aggregationMode.ordinal()].getCalendarValue(); + + Calendar alignedStartTime = getAlignedStartTime(); + logger.trace("Aligned StartTime : {}", timeInMillisToString(alignedStartTime.getTimeInMillis())); + + Calendar alignedEndTime = getAlignedEndTime(); + long alignedEndTimeInMillis = alignedEndTime.getTimeInMillis(); + logger.trace("Aligned EndTime : {}", timeInMillisToString(alignedEndTime.getTimeInMillis())); + + Calendar progressTime = Calendar.getInstance(); + progressTime.setTimeZone(DEFAULT_TIME_ZONE); + progressTime.setTimeInMillis(alignedStartTime.getTimeInMillis()); + + while(progressTime.getTimeInMillis() <= alignedEndTimeInMillis){ + logger.trace("Progress Time : {}", timeInMillisToString(progressTime.getTimeInMillis())); + Calendar item = Calendar.getInstance(); + item.setTimeZone(DEFAULT_TIME_ZONE); + item.setTimeInMillis(progressTime.getTimeInMillis()); + + sequence.add(item); + progressTime.add(calendarValue, 1); + } + + return sequence; + } + + + public static List getSequenceAsStringList(List sequence){ + List stringSequence = new ArrayList(); + for(Calendar calendar : sequence){ + stringSequence.add(timeInMillisToString(calendar.getTimeInMillis())); + } + return stringSequence; + } + + @Override + public String toString(){ + return String.format("StartTime : %s, EndTime : %s, Aggregated %s", + timeInMillisToString(startTime), + timeInMillisToString(endTime), + aggregationMode.toString()); + } } diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceQuery.java b/src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceQuery.java index 544a000..ded3e1b 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceQuery.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceQuery.java @@ -3,7 +3,9 @@ */ package org.gcube.accounting.analytics.persistence; +import java.util.Calendar; import java.util.List; +import java.util.Map; import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Info; @@ -18,10 +20,10 @@ public abstract class AccountingPersistenceQuery { protected abstract void prepareConnection(AccountingPersistenceQueryConfiguration configuration) throws Exception; - protected abstract List reallyQuery(Class usageRecordType, + protected abstract Map reallyQuery(Class usageRecordType, TemporalConstraint temporalConstraint, List filters) throws Exception; - public List query(Class usageRecordType, + public Map query(Class usageRecordType, TemporalConstraint temporalConstraint, List filters) throws Exception{ return reallyQuery(usageRecordType, temporalConstraint, filters); } diff --git a/src/test/java/org/gcube/accounting/analytics/TemporalConstraintTest.java b/src/test/java/org/gcube/accounting/analytics/TemporalConstraintTest.java new file mode 100644 index 0000000..f526209 --- /dev/null +++ b/src/test/java/org/gcube/accounting/analytics/TemporalConstraintTest.java @@ -0,0 +1,109 @@ +/** + * + */ +package org.gcube.accounting.analytics; + +import java.util.Calendar; +import java.util.List; + +import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ + * + */ +public class TemporalConstraintTest { + + private static Logger logger = LoggerFactory.getLogger(TemporalConstraintTest.class); + + + long testTime = new Long("1437481211396"); // 2015-07-21 12:20:11:396 UTC + long[] results = new long[]{ + new Long("1420070400000"), // 2015-01-01 00:00:00:00 UTC + new Long("1435708800000"), // 2015-07-01 00:00:00:00 UTC + new Long("1437436800000"), // 2015-07-21 00:00:00:00 UTC + new Long("1437480000000"), // 2015-07-21 12:00:00:00 UTC + new Long("1437481200000"), // 2015-07-21 12:20:00:00 UTC + new Long("1437481211000"), // 2015-07-21 12:20:11:00 UTC + testTime // 2015-07-21 12:20:11:396 UTC + }; + + @Test + public void getAlignedCalendarTest(){ + logger.debug(TemporalConstraint.timeInMillisToString(testTime)); + for(AggregationMode aggregationMode : AggregationMode.values()){ + Calendar alignedCalendar = TemporalConstraint.getAlignedCalendar(testTime, aggregationMode); + long alignedInMillis = alignedCalendar.getTimeInMillis(); + logger.debug("With AggregationMode {} The aligned value of {} is {}", + aggregationMode, + TemporalConstraint.timeInMillisToString(testTime), + TemporalConstraint.timeInMillisToString(alignedInMillis)); + Assert.assertEquals(results[aggregationMode.ordinal()], alignedInMillis); + } + + } + + @Test + public void getCalendarSequenceTest(){ + Calendar endTime = Calendar.getInstance(); + + Calendar startTime = Calendar.getInstance(); + // 3 days + int[] timeShift = new int[]{ + 3, // days + 24, // hour in a day + 60, // min per hour + 60, // sec per min + 1000 // millisec per sec + }; + + long shift = 1; + for(int i=0; i sequence = temporalConstraint.getCalendarSequence(); + if(aggregationMode.ordinal()<=AggregationMode.HOURLY.ordinal()){ + logger.debug("{} generate the following sequence (size {}) {}", + temporalConstraint, sequence.size(), + TemporalConstraint.getSequenceAsStringList(sequence)); + }else{ + logger.debug("{} generate asequence with size {}", + temporalConstraint, sequence.size()); + } + int expected = expectedSequenceSize[aggregationMode.ordinal()]; + + // Expected has 1 more value because the extremes are contained. + // DAILY because the difference is 3 DAYS + if(aggregationMode.ordinal()>=AggregationMode.DAILY.ordinal()){ + expected++; + } + + Assert.assertEquals(expected, sequence.size()); + } + } + + + +}