refs #2253: Accounting Manager - Top N Active Users
https://support.d4science.org/issues/2253 Reorganized Library git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics@125496 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
31e6f90591
commit
79c9373eae
|
|
@ -1,183 +0,0 @@
|
|||
/**
|
||||
*
|
||||
*/
|
||||
package org.gcube.accounting.analytics;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.accounting.analytics.exception.NoAvailableScopeException;
|
||||
import org.gcube.accounting.analytics.exception.NoUsableAccountingPersistenceQueryFound;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryFactory;
|
||||
import org.gcube.documentstore.records.AggregatedRecord;
|
||||
import org.gcube.documentstore.records.Record;
|
||||
import org.gcube.documentstore.records.RecordUtility;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
|
||||
*/
|
||||
public class ResourceRecordQuery {
|
||||
|
||||
private static Logger logger = LoggerFactory
|
||||
.getLogger(ResourceRecordQuery.class);
|
||||
|
||||
protected static Map<Class<? extends Record>, Set<String>> resourceRecords = null;
|
||||
|
||||
/**
|
||||
* Return a Map containing a set of required fields for each Resource
|
||||
* Records Types
|
||||
*
|
||||
* @return the Map
|
||||
*/
|
||||
public static synchronized Map<Class<? extends Record>, Set<String>> getResourceRecordsTypes() {
|
||||
if (resourceRecords == null) {
|
||||
resourceRecords = new HashMap<Class<? extends Record>, Set<String>>();
|
||||
Collection<Class<? extends Record>> resourceRecordsTypes = RecordUtility
|
||||
.getRecordClassesFound().values();
|
||||
for (Class<? extends Record> resourceRecordsType : resourceRecordsTypes) {
|
||||
try {
|
||||
Record record = resourceRecordsType.newInstance();
|
||||
resourceRecords.put(resourceRecordsType,
|
||||
record.getRequiredFields());
|
||||
} catch (InstantiationException | IllegalAccessException e) {
|
||||
logger.error(String.format(
|
||||
"Unable to correctly istantiate %s",
|
||||
resourceRecordsType.getSimpleName()), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return resourceRecords;
|
||||
}
|
||||
|
||||
protected AccountingPersistenceBackendQuery accountingPersistenceQuery;
|
||||
|
||||
/**
|
||||
* Instantiate the ResourceRecord for the current scope
|
||||
*
|
||||
* @throws NoAvailableScopeException
|
||||
* if there is not possible to query in the current scope
|
||||
* @throws NoUsableAccountingPersistenceQueryFound
|
||||
* if there is no available instance which can query in that
|
||||
* scope
|
||||
*/
|
||||
public ResourceRecordQuery() throws NoAvailableScopeException,
|
||||
NoUsableAccountingPersistenceQueryFound {
|
||||
this.accountingPersistenceQuery = AccountingPersistenceBackendQueryFactory
|
||||
.getInstance();
|
||||
}
|
||||
|
||||
protected static JSONObject getPaddingJSONObject(
|
||||
Map<Calendar, Info> unpaddedResults) throws JSONException {
|
||||
Info auxInfo = new ArrayList<Info>(unpaddedResults.values()).get(0);
|
||||
JSONObject auxJsonObject = auxInfo.getValue();
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<String> keys = auxJsonObject.keys();
|
||||
|
||||
JSONObject jsonObject = new JSONObject();
|
||||
while (keys.hasNext()) {
|
||||
String key = keys.next();
|
||||
jsonObject.put(key, 0);
|
||||
}
|
||||
|
||||
return jsonObject;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pad the data
|
||||
*
|
||||
* @param unpaddedData
|
||||
* the data to be pad
|
||||
* @param temporalConstraint
|
||||
* temporalConstraint the temporal interval and the granularity
|
||||
* of the data to pad
|
||||
* @return the data padded taking in account the TemporalConstraint
|
||||
* @throws Exception
|
||||
* if fails
|
||||
*/
|
||||
public static List<Info> getPaddedResults(Map<Calendar, Info> unpaddedData,
|
||||
TemporalConstraint temporalConstraint) throws Exception {
|
||||
JSONObject jsonObject = getPaddingJSONObject(unpaddedData);
|
||||
|
||||
List<Info> paddedResults = new ArrayList<Info>();
|
||||
List<Calendar> sequence = temporalConstraint.getCalendarSequence();
|
||||
|
||||
for (Calendar progressTime : sequence) {
|
||||
if (unpaddedData.get(progressTime) != null) {
|
||||
paddedResults.add(unpaddedData.get(progressTime));
|
||||
} else {
|
||||
Info info = new Info(progressTime, jsonObject);
|
||||
paddedResults.add(info);
|
||||
}
|
||||
}
|
||||
|
||||
return paddedResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return results with padding if pad is set to true.
|
||||
*
|
||||
* @param aggregatedRecordClass
|
||||
* the UsageRecord type to query
|
||||
* @param temporalConstraint
|
||||
* the temporal interval and the granularity
|
||||
* @param filters
|
||||
* the list keys to filter (in AND)
|
||||
* @param pad
|
||||
* indicate is the results have to be padded with zeros when
|
||||
* there is no data available at certain data points of sequence
|
||||
* @return the requested list of Info
|
||||
* @throws Exception
|
||||
* if fails
|
||||
*/
|
||||
public List<Info> getInfo(
|
||||
Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters,
|
||||
boolean pad) throws Exception {
|
||||
|
||||
Map<Calendar, Info> unpaddedResults = accountingPersistenceQuery
|
||||
.getTimeSeries(aggregatedRecordClass, temporalConstraint,
|
||||
filters);
|
||||
|
||||
if (!pad) {
|
||||
return new ArrayList<Info>(unpaddedResults.values());
|
||||
}
|
||||
|
||||
return getPaddedResults(unpaddedResults, temporalConstraint);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return unpadded results
|
||||
*
|
||||
* @param aggregatedRecordClass
|
||||
* the UsageRecord type to query
|
||||
* @param temporalConstraint
|
||||
* the temporal interval and the granularity
|
||||
* @param filters
|
||||
* the list keys to filter (in AND)
|
||||
* @return the requested list of Info
|
||||
* @throws Exception
|
||||
* if fails
|
||||
*/
|
||||
public List<Info> getInfo(
|
||||
Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters)
|
||||
throws Exception {
|
||||
|
||||
return getInfo(aggregatedRecordClass, temporalConstraint, filters,
|
||||
false);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -6,9 +6,12 @@ package org.gcube.accounting.analytics;
|
|||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TimeZone;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -152,8 +155,8 @@ public class TemporalConstraint {
|
|||
return getAlignedCalendar(endTime, aggregationMode);
|
||||
}
|
||||
|
||||
public List<Calendar> getCalendarSequence(){
|
||||
List<Calendar> sequence = new ArrayList<Calendar>();
|
||||
public SortedSet<Calendar> getCalendarSequence(){
|
||||
SortedSet<Calendar> sequence = new TreeSet<>();
|
||||
|
||||
CalendarEnum[] calendarValues = CalendarEnum.values();
|
||||
int calendarValue = calendarValues[aggregationMode.ordinal()].getCalendarValue();
|
||||
|
|
@ -183,7 +186,7 @@ public class TemporalConstraint {
|
|||
}
|
||||
|
||||
|
||||
public static List<String> getSequenceAsStringList(List<Calendar> sequence){
|
||||
public static List<String> getSequenceAsStringList(Collection<Calendar> sequence){
|
||||
List<String> stringSequence = new ArrayList<String>();
|
||||
for(Calendar calendar : sequence){
|
||||
stringSequence.add(timeInMillisToString(calendar.getTimeInMillis()));
|
||||
|
|
|
|||
|
|
@ -3,8 +3,11 @@
|
|||
*/
|
||||
package org.gcube.accounting.analytics.persistence;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.SortedMap;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
|
|
@ -17,6 +20,8 @@ import org.gcube.accounting.datamodel.UsageRecord;
|
|||
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
|
||||
import org.gcube.documentstore.records.AggregatedRecord;
|
||||
import org.gcube.documentstore.records.Record;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
/**
|
||||
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
|
||||
|
|
@ -26,6 +31,8 @@ public class AccountingPersistenceQuery {
|
|||
|
||||
private static final AccountingPersistenceQuery accountingPersistenceQuery;
|
||||
|
||||
public static final int DEFAULT_LIMIT_RESULT_NUMBER = 5;
|
||||
|
||||
private AccountingPersistenceQuery() {
|
||||
}
|
||||
|
||||
|
|
@ -62,11 +69,18 @@ public class AccountingPersistenceQuery {
|
|||
|
||||
public SortedMap<Calendar, Info> getTimeSeries(
|
||||
Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters)
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters, boolean pad)
|
||||
throws Exception {
|
||||
return AccountingPersistenceBackendQueryFactory.getInstance()
|
||||
SortedMap<Calendar, Info> ret =
|
||||
AccountingPersistenceBackendQueryFactory.getInstance()
|
||||
.getTimeSeries(aggregatedRecordClass, temporalConstraint,
|
||||
filters);
|
||||
|
||||
if(pad){
|
||||
ret = padMap(ret, temporalConstraint);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static String getDefaultOrderingProperties(Class<? extends AggregatedRecord<?, ?>> recordClass){
|
||||
|
|
@ -76,8 +90,77 @@ public class AccountingPersistenceQuery {
|
|||
return AggregatedRecord.OPERATION_COUNT;
|
||||
}
|
||||
|
||||
protected static JSONObject getPaddingJSONObject(
|
||||
Map<Calendar, Info> unpaddedResults) throws JSONException {
|
||||
Info auxInfo = new ArrayList<Info>(unpaddedResults.values()).get(0);
|
||||
JSONObject auxJsonObject = auxInfo.getValue();
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<String> keys = auxJsonObject.keys();
|
||||
|
||||
JSONObject jsonObject = new JSONObject();
|
||||
while (keys.hasNext()) {
|
||||
String key = keys.next();
|
||||
jsonObject.put(key, 0);
|
||||
}
|
||||
|
||||
return jsonObject;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pad the data
|
||||
*
|
||||
* @param unpaddedData
|
||||
* the data to be pad
|
||||
* @param temporalConstraint
|
||||
* temporalConstraint the temporal interval and the granularity
|
||||
* of the data to pad
|
||||
* @return the data padded taking in account the TemporalConstraint
|
||||
* @throws Exception
|
||||
* if fails
|
||||
*/
|
||||
public static SortedMap<Calendar, Info> padMap(
|
||||
SortedMap<Calendar, Info> unpaddedData,
|
||||
TemporalConstraint temporalConstraint) throws Exception {
|
||||
|
||||
JSONObject jsonObject = getPaddingJSONObject(unpaddedData);
|
||||
SortedSet<Calendar> sequence = temporalConstraint.getCalendarSequence();
|
||||
for (Calendar progressTime : sequence) {
|
||||
Info info = unpaddedData.get(progressTime);
|
||||
if (info == null) {
|
||||
info = new Info(progressTime, jsonObject);
|
||||
unpaddedData.put(progressTime, info);
|
||||
}
|
||||
}
|
||||
return unpaddedData;
|
||||
}
|
||||
|
||||
public static SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues(
|
||||
Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters,
|
||||
String orderingProperty, boolean pad, int limit) throws Exception {
|
||||
|
||||
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> ret;
|
||||
|
||||
if(orderingProperty==null){
|
||||
orderingProperty = getDefaultOrderingProperties(aggregatedRecordClass);
|
||||
}
|
||||
|
||||
ret = AccountingPersistenceBackendQueryFactory.getInstance()
|
||||
.getTopValues(aggregatedRecordClass, temporalConstraint,
|
||||
filters, orderingProperty);
|
||||
|
||||
|
||||
if(pad){
|
||||
int count = ret.size() > limit ? limit : ret.size();
|
||||
while(--count >= 0){
|
||||
for(NumberedFilter nf : ret.keySet()){
|
||||
padMap(ret.get(nf), temporalConstraint);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues(
|
||||
Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
|
||||
|
|
@ -96,7 +179,8 @@ public class AccountingPersistenceQuery {
|
|||
.getNextPossibleValues(aggregatedRecordClass,
|
||||
temporalConstraint, filters);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Close the connection to persistence
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
/**
|
||||
*
|
||||
*/
|
||||
package org.gcube.accounting.analytics;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.accounting.datamodel.BasicUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.PortletUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.TaskUsageRecord;
|
||||
import org.gcube.documentstore.records.Record;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
|
||||
*/
|
||||
public class ResourceRecordQueryTest {
|
||||
|
||||
public class TestUsageRecord extends BasicUsageRecord {
|
||||
|
||||
/**
|
||||
* Generated Serial Version UID
|
||||
*/
|
||||
private static final long serialVersionUID = 1939161386352514727L;
|
||||
|
||||
@Override
|
||||
protected String giveMeRecordType() {
|
||||
return TestUsageRecord.class.getSimpleName();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static Set<Class<? extends Record>> getExpectedResourceRecordsTypes(){
|
||||
Set<Class<? extends Record>> expected = new HashSet<Class<? extends Record>>();
|
||||
expected.add(ServiceUsageRecord.class);
|
||||
expected.add(StorageUsageRecord.class);
|
||||
expected.add(JobUsageRecord.class);
|
||||
expected.add(TaskUsageRecord.class);
|
||||
expected.add(PortletUsageRecord.class);
|
||||
return expected;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetResourceRecordsTypes(){
|
||||
Set<Class<? extends Record>> expected = getExpectedResourceRecordsTypes();
|
||||
Set<Class<? extends Record>> found = ResourceRecordQuery.getResourceRecordsTypes().keySet();
|
||||
Assert.assertTrue(expected.containsAll(found));
|
||||
Assert.assertTrue(found.containsAll(expected));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetResourceRecordsTypesWithFakeClass(){
|
||||
Set<Class<? extends Record>> expected = getExpectedResourceRecordsTypes();
|
||||
expected.add(TestUsageRecord.class);
|
||||
Set<Class<? extends Record>> found = ResourceRecordQuery.getResourceRecordsTypes().keySet();
|
||||
Assert.assertTrue(expected.containsAll(found));
|
||||
Assert.assertFalse(found.containsAll(expected));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
package org.gcube.accounting.analytics;
|
||||
|
||||
import java.util.Calendar;
|
||||
import java.util.List;
|
||||
import java.util.Collection;
|
||||
|
||||
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||
import org.junit.Assert;
|
||||
|
|
@ -82,7 +82,7 @@ public class TemporalConstraintTest {
|
|||
break;
|
||||
}
|
||||
|
||||
List<Calendar> sequence = temporalConstraint.getCalendarSequence();
|
||||
Collection<Calendar> sequence = temporalConstraint.getCalendarSequence();
|
||||
if(aggregationMode.ordinal()<=AggregationMode.HOURLY.ordinal()){
|
||||
logger.debug("{} generate the following sequence (size {}) {}",
|
||||
temporalConstraint, sequence.size(),
|
||||
|
|
|
|||
Loading…
Reference in New Issue