gCubeSystem
/
accounting-analytics
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implemented padded results and related functionality 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics@117390 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2015-07-21 14:33:36 +00:00
parent 500e578fb1
commit 1e78ae0fcd
7 changed files with 306 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pom.xml
View File

@ -65,6 +65,12 @@
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.13</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

42
src/main/java/org/gcube/accounting/analytics/BasicQuery.java
View File

@ -1,42 +0,0 @@
/**
*
*/
package org.gcube.accounting.analytics;
import java.util.List;
import org.gcube.accounting.analytics.exception.NoAvailableScopeException;
import org.gcube.accounting.analytics.exception.NoUsableAccountingPersistenceQueryFound;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
import org.gcube.common.scope.api.ScopeProvider;
/**
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
*/
public class BasicQuery {
protected AccountingPersistenceQuery accountingPersistenceQuery;
public BasicQuery() throws NoAvailableScopeException, NoUsableAccountingPersistenceQueryFound {
this.accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
}
public BasicQuery(String scope) throws NoAvailableScopeException, NoUsableAccountingPersistenceQueryFound {
ScopeProvider.instance.set(scope);
this.accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
}
public List<Info> getServiceInfo(TemporalConstraint temporalConstraint) {
return null;
}
public List<Info> getStorageInfo(TemporalConstraint temporalConstraint) {
return null;
}
}

2
src/main/java/org/gcube/accounting/analytics/Info.java
View File

@ -60,7 +60,7 @@ public class Info {
@Override
public String toString(){
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SS z");
simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
simpleDateFormat.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
return String.format("Date : %s, Value : %s", simpleDateFormat.format(date), value.toString());
}
}

70
src/main/java/org/gcube/accounting/analytics/ResourceRecordQuery.java
View File

@ -3,7 +3,11 @@
*/
package org.gcube.accounting.analytics;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -15,6 +19,8 @@ import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFact
import org.gcube.accounting.datamodel.SingleUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.common.scope.api.ScopeProvider;
import org.json.JSONException;
import org.json.JSONObject;
import org.reflections.Reflections;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -59,14 +65,72 @@ public class ResourceRecordQuery {
this.accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
}
public List<Info> getInfo(Class<? extends SingleUsageRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{
return accountingPersistenceQuery.query(usageRecordType, temporalConstraint, filters);
protected static JSONObject getPaddingJSONObject(Map<Calendar, Info> unpaddedResults) throws JSONException{
Info auxInfo = new ArrayList<Info>(unpaddedResults.values()).get(0);
JSONObject auxJsonObject = auxInfo.getValue();
@SuppressWarnings("unchecked")
Iterator<String> keys = auxJsonObject.keys();
JSONObject jsonObject = new JSONObject();
while(keys.hasNext()){
String key = keys.next();
jsonObject.put(key, 0);
}
return jsonObject;
}
public static List<Info> getPaddedResults(Map<Calendar, Info> unpaddedResults, TemporalConstraint temporalConstraint) throws JSONException{
JSONObject jsonObject = getPaddingJSONObject(unpaddedResults);
List<Info> paddedResults = new ArrayList<Info>();
List<Calendar> sequence = temporalConstraint.getCalendarSequence();
for(Calendar progressTime : sequence){
if(unpaddedResults.get(progressTime)!=null){
paddedResults.add(unpaddedResults.get(progressTime));
}else{
Date date = new Date(progressTime.getTimeInMillis());
Info info = new Info(date, jsonObject);
paddedResults.add(info);
}
}
return paddedResults;
}
/**
* Return results with padding if pad is set to true.
* @param usageRecordType
* @param temporalConstraint
* @param filters
* @param pad
* @return
* @throws Exception
*/
public List<Info> getInfo(Class<? extends SingleUsageRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters, boolean pad) throws Exception {
Map<Calendar, Info> unpaddedResults = accountingPersistenceQuery.query(usageRecordType, temporalConstraint, filters);
if(!pad){
return new ArrayList<Info>(unpaddedResults.values());
}
return getPaddedResults(unpaddedResults, temporalConstraint);
}
/**
* Return unpadded result
* @param usageRecordType
* @param temporalConstraint
* @param filters
* @return
* @throws Exception
*/
public List<Info> getInfo(Class<? extends SingleUsageRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{
return getInfo(usageRecordType, temporalConstraint, filters, false);
}
}

120
src/main/java/org/gcube/accounting/analytics/TemporalConstraint.java
View File

@ -3,16 +3,52 @@
*/
package org.gcube.accounting.analytics;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
*
*/
public class TemporalConstraint {
private static final Logger logger = LoggerFactory.getLogger(TemporalConstraint.class);
private static final String UTC_TIME_ZONE = "UTC";
public static final TimeZone DEFAULT_TIME_ZONE = TimeZone.getTimeZone(UTC_TIME_ZONE);
public enum AggregationMode {
YEARLY, MONTHLY, DAILY, HOURLY, MINUTELY, SECONDLY, MILLISECONDLY
}
public enum CalendarEnum {
YEAR(Calendar.YEAR),
MONTH(Calendar.MONTH),
DAY(Calendar.DAY_OF_MONTH),
HOUR(Calendar.HOUR_OF_DAY),
MINUTE(Calendar.MINUTE),
SECOND(Calendar.SECOND),
MILLISECOND(Calendar.MILLISECOND);
private final int calendarValue;
CalendarEnum(int calendarValue){
this.calendarValue = calendarValue;
}
public int getCalendarValue(){
return calendarValue;
}
};
protected long startTime;
protected long endTime;
protected AggregationMode aggregationMode;
@ -64,6 +100,88 @@ public class TemporalConstraint {
public void setAggregationMode(AggregationMode aggregationMode) {
this.aggregationMode = aggregationMode;
}
public static String timeInMillisToString(long timeInMillis){
Date date = new Date(timeInMillis);
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SS z");
simpleDateFormat.setTimeZone(DEFAULT_TIME_ZONE);
return String.format("%s (%d millis)", simpleDateFormat.format(date), timeInMillis);
}
public static Calendar getAlignedCalendar(long millis, AggregationMode aggregationMode){
Calendar alignedCalendar = Calendar.getInstance();
alignedCalendar.setTimeZone(DEFAULT_TIME_ZONE);
alignedCalendar.setTimeInMillis(millis);
CalendarEnum[] calendarValues = CalendarEnum.values();
for(int i=aggregationMode.ordinal()+1; i<calendarValues.length; i++){
int calendarValue = calendarValues[i].getCalendarValue();
if(calendarValue == Calendar.DAY_OF_MONTH){
alignedCalendar.set(calendarValue, 1);
} else{
alignedCalendar.set(calendarValue, 0);
}
}
return alignedCalendar;
}
public Calendar getAlignedStartTime(){
return getAlignedCalendar(startTime, aggregationMode);
}
public Calendar getAlignedEndTime(){
return getAlignedCalendar(endTime, aggregationMode);
}
public List<Calendar> getCalendarSequence(){
List<Calendar> sequence = new ArrayList<Calendar>();
CalendarEnum[] calendarValues = CalendarEnum.values();
int calendarValue = calendarValues[aggregationMode.ordinal()].getCalendarValue();
Calendar alignedStartTime = getAlignedStartTime();
logger.trace("Aligned StartTime : {}", timeInMillisToString(alignedStartTime.getTimeInMillis()));
Calendar alignedEndTime = getAlignedEndTime();
long alignedEndTimeInMillis = alignedEndTime.getTimeInMillis();
logger.trace("Aligned EndTime : {}", timeInMillisToString(alignedEndTime.getTimeInMillis()));
Calendar progressTime = Calendar.getInstance();
progressTime.setTimeZone(DEFAULT_TIME_ZONE);
progressTime.setTimeInMillis(alignedStartTime.getTimeInMillis());
while(progressTime.getTimeInMillis() <= alignedEndTimeInMillis){
logger.trace("Progress Time : {}", timeInMillisToString(progressTime.getTimeInMillis()));
Calendar item = Calendar.getInstance();
item.setTimeZone(DEFAULT_TIME_ZONE);
item.setTimeInMillis(progressTime.getTimeInMillis());
sequence.add(item);
progressTime.add(calendarValue, 1);
}
return sequence;
}
public static List<String> getSequenceAsStringList(List<Calendar> sequence){
List<String> stringSequence = new ArrayList<String>();
for(Calendar calendar : sequence){
stringSequence.add(timeInMillisToString(calendar.getTimeInMillis()));
}
return stringSequence;
}
@Override
public String toString(){
return String.format("StartTime : %s, EndTime : %s, Aggregated %s",
timeInMillisToString(startTime),
timeInMillisToString(endTime),
aggregationMode.toString());
}
}

6
src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceQuery.java
View File

@ -3,7 +3,9 @@
*/
package org.gcube.accounting.analytics.persistence;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info;
@ -18,10 +20,10 @@ public abstract class AccountingPersistenceQuery {
protected abstract void prepareConnection(AccountingPersistenceQueryConfiguration configuration) throws Exception;
protected abstract List<Info> reallyQuery(Class<? extends SingleUsageRecord> usageRecordType,
protected abstract Map<Calendar, Info> reallyQuery(Class<? extends SingleUsageRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception;
public List<Info> query(Class<? extends SingleUsageRecord> usageRecordType,
public Map<Calendar, Info> query(Class<? extends SingleUsageRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{
return reallyQuery(usageRecordType, temporalConstraint, filters);
}

109
src/test/java/org/gcube/accounting/analytics/TemporalConstraintTest.java Normal file
View File

@ -0,0 +1,109 @@
/**
*
*/
package org.gcube.accounting.analytics;
import java.util.Calendar;
import java.util.List;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
*
*/
public class TemporalConstraintTest {
private static Logger logger = LoggerFactory.getLogger(TemporalConstraintTest.class);
long testTime = new Long("1437481211396"); // 2015-07-21 12:20:11:396 UTC
long[] results = new long[]{
new Long("1420070400000"), // 2015-01-01 00:00:00:00 UTC
new Long("1435708800000"), // 2015-07-01 00:00:00:00 UTC
new Long("1437436800000"), // 2015-07-21 00:00:00:00 UTC
new Long("1437480000000"), // 2015-07-21 12:00:00:00 UTC
new Long("1437481200000"), // 2015-07-21 12:20:00:00 UTC
new Long("1437481211000"), // 2015-07-21 12:20:11:00 UTC
testTime // 2015-07-21 12:20:11:396 UTC
};
@Test
public void getAlignedCalendarTest(){
logger.debug(TemporalConstraint.timeInMillisToString(testTime));
for(AggregationMode aggregationMode : AggregationMode.values()){
Calendar alignedCalendar = TemporalConstraint.getAlignedCalendar(testTime, aggregationMode);
long alignedInMillis = alignedCalendar.getTimeInMillis();
logger.debug("With AggregationMode {} The aligned value of {} is {}",
aggregationMode,
TemporalConstraint.timeInMillisToString(testTime),
TemporalConstraint.timeInMillisToString(alignedInMillis));
Assert.assertEquals(results[aggregationMode.ordinal()], alignedInMillis);
}
}
@Test
public void getCalendarSequenceTest(){
Calendar endTime = Calendar.getInstance();
Calendar startTime = Calendar.getInstance();
// 3 days
int[] timeShift = new int[]{
3, // days
24, // hour in a day
60, // min per hour
60, // sec per min
1000 // millisec per sec
};
long shift = 1;
for(int i=0; i<timeShift.length; i++){
shift = shift * timeShift[i];
}
startTime.setTimeInMillis(endTime.getTimeInMillis()-(shift));
int[] expectedSequenceSize = new int[AggregationMode.values().length];
expectedSequenceSize[0] = 1; // AggregationMode.YEARLY
expectedSequenceSize[1] = 1; // AggregationMode.MONTHLY
for(int j=0; j<timeShift.length; j++){
expectedSequenceSize[j+2] = expectedSequenceSize[j+1] * timeShift[j];
};
for(AggregationMode aggregationMode : AggregationMode.values()){
TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), aggregationMode);
if(aggregationMode==AggregationMode.MILLISECONDLY ||
aggregationMode==AggregationMode.SECONDLY ||
aggregationMode==AggregationMode.MINUTELY){
break;
}
List<Calendar> sequence = temporalConstraint.getCalendarSequence();
if(aggregationMode.ordinal()<=AggregationMode.HOURLY.ordinal()){
logger.debug("{} generate the following sequence (size {}) {}",
temporalConstraint, sequence.size(),
TemporalConstraint.getSequenceAsStringList(sequence));
}else{
logger.debug("{} generate asequence with size {}",
temporalConstraint, sequence.size());
}
int expected = expectedSequenceSize[aggregationMode.ordinal()];
// Expected has 1 more value because the extremes are contained.
// DAILY because the difference is 3 DAYS
if(aggregationMode.ordinal()>=AggregationMode.DAILY.ordinal()){
expected++;
}
Assert.assertEquals(expected, sequence.size());
}
}
}