Reorganizing library

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics@125439 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2016-03-16 12:29:03 +00:00
parent d3dab39dfb
commit e8d8bc0f09
5 changed files with 268 additions and 177 deletions

View File

@ -9,7 +9,7 @@
<groupId>org.gcube.accounting</groupId> <groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics</artifactId> <artifactId>accounting-analytics</artifactId>
<version>1.2.0-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<name>accounting-analytics</name> <name>accounting-analytics</name>
<scm> <scm>

View File

@ -7,7 +7,7 @@ package org.gcube.accounting.analytics;
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
* *
*/ */
public class Filter { public class Filter implements Comparable<Filter> {
protected String key; protected String key;
protected String value; protected String value;
@ -49,4 +49,53 @@ public class Filter {
this.value = value; this.value = value;
} }
@Override
public String toString(){
return String.format("{ \"%s\" : \"%s\" }", key, value);
}
/** {@inheritDoc} */
@Override
public int compareTo(Filter filter) {
int compareResult = this.key.compareTo(filter.key);
if(compareResult == 0){
compareResult = this.value.compareTo(filter.value);
}
return compareResult;
}
/** {@inheritDoc} */
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((key == null) ? 0 : key.hashCode());
result = prime * result + ((value == null) ? 0 : value.hashCode());
return result;
}
/** {@inheritDoc} */
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
Filter other = (Filter) obj;
if (key == null) {
if (other.key != null)
return false;
} else if (!key.equals(other.key))
return false;
if (value == null) {
if (other.value != null)
return false;
} else if (!value.equals(other.value))
return false;
return true;
}
} }

View File

@ -6,7 +6,6 @@ package org.gcube.accounting.analytics;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
@ -29,139 +28,156 @@ import org.slf4j.LoggerFactory;
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
*/ */
public class ResourceRecordQuery { public class ResourceRecordQuery {
private static Logger logger = LoggerFactory.getLogger(ResourceRecordQuery.class); private static Logger logger = LoggerFactory
.getLogger(ResourceRecordQuery.class);
protected static Map<Class<? extends Record>, Set<String>> resourceRecords = null; protected static Map<Class<? extends Record>, Set<String>> resourceRecords = null;
/** /**
* Return a Map containing a set of required fields for each Resource * Return a Map containing a set of required fields for each Resource
* Records Types * Records Types
*
* @return the Map * @return the Map
*/ */
public static synchronized Map<Class<? extends Record>, Set<String>> getResourceRecordsTypes() { public static synchronized Map<Class<? extends Record>, Set<String>> getResourceRecordsTypes() {
if(resourceRecords==null){ if (resourceRecords == null) {
resourceRecords = new HashMap<Class<? extends Record>, Set<String>>(); resourceRecords = new HashMap<Class<? extends Record>, Set<String>>();
Collection<Class<? extends Record>> resourceRecordsTypes = RecordUtility.getRecordClassesFound().values(); Collection<Class<? extends Record>> resourceRecordsTypes = RecordUtility
for(Class<? extends Record> resourceRecordsType : resourceRecordsTypes){ .getRecordClassesFound().values();
for (Class<? extends Record> resourceRecordsType : resourceRecordsTypes) {
try { try {
Record record = resourceRecordsType.newInstance(); Record record = resourceRecordsType.newInstance();
resourceRecords.put(resourceRecordsType, record.getRequiredFields()); resourceRecords.put(resourceRecordsType,
record.getRequiredFields());
} catch (InstantiationException | IllegalAccessException e) { } catch (InstantiationException | IllegalAccessException e) {
logger.error(String.format("Unable to correctly istantiate %s", resourceRecordsType.getSimpleName()), e); logger.error(String.format(
"Unable to correctly istantiate %s",
resourceRecordsType.getSimpleName()), e);
} }
} }
} }
return resourceRecords; return resourceRecords;
} }
protected AccountingPersistenceBackendQuery accountingPersistenceQuery; protected AccountingPersistenceBackendQuery accountingPersistenceQuery;
/** /**
* Instantiate the ResourceRecord for the current scope * Instantiate the ResourceRecord for the current scope
* @throws NoAvailableScopeException if there is not possible to query in *
* the current scope * @throws NoAvailableScopeException
* @throws NoUsableAccountingPersistenceQueryFound if there is no available * if there is not possible to query in the current scope
* instance which can query in that scope * @throws NoUsableAccountingPersistenceQueryFound
* if there is no available instance which can query in that
* scope
*/ */
public ResourceRecordQuery() throws NoAvailableScopeException, NoUsableAccountingPersistenceQueryFound { public ResourceRecordQuery() throws NoAvailableScopeException,
this.accountingPersistenceQuery = AccountingPersistenceBackendQueryFactory.getInstance(); NoUsableAccountingPersistenceQueryFound {
this.accountingPersistenceQuery = AccountingPersistenceBackendQueryFactory
.getInstance();
} }
protected static JSONObject getPaddingJSONObject(Map<Calendar, Info> unpaddedResults) throws JSONException{ protected static JSONObject getPaddingJSONObject(
Map<Calendar, Info> unpaddedResults) throws JSONException {
Info auxInfo = new ArrayList<Info>(unpaddedResults.values()).get(0); Info auxInfo = new ArrayList<Info>(unpaddedResults.values()).get(0);
JSONObject auxJsonObject = auxInfo.getValue(); JSONObject auxJsonObject = auxInfo.getValue();
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
Iterator<String> keys = auxJsonObject.keys(); Iterator<String> keys = auxJsonObject.keys();
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
while(keys.hasNext()){ while (keys.hasNext()) {
String key = keys.next(); String key = keys.next();
jsonObject.put(key, 0); jsonObject.put(key, 0);
} }
return jsonObject; return jsonObject;
} }
/** /**
* Pad the data * Pad the data
* @param unpaddedData the data to be pad *
* @param temporalConstraint temporalConstraint the temporal interval and * @param unpaddedData
* the granularity of the data to pad * the data to be pad
* @param temporalConstraint
* temporalConstraint the temporal interval and the granularity
* of the data to pad
* @return the data padded taking in account the TemporalConstraint * @return the data padded taking in account the TemporalConstraint
* @throws Exception if fails * @throws Exception
* if fails
*/ */
public static List<Info> getPaddedResults(Map<Calendar, Info> unpaddedData, public static List<Info> getPaddedResults(Map<Calendar, Info> unpaddedData,
TemporalConstraint temporalConstraint) throws Exception { TemporalConstraint temporalConstraint) throws Exception {
JSONObject jsonObject = getPaddingJSONObject(unpaddedData); JSONObject jsonObject = getPaddingJSONObject(unpaddedData);
List<Info> paddedResults = new ArrayList<Info>(); List<Info> paddedResults = new ArrayList<Info>();
List<Calendar> sequence = temporalConstraint.getCalendarSequence(); List<Calendar> sequence = temporalConstraint.getCalendarSequence();
for(Calendar progressTime : sequence){ for (Calendar progressTime : sequence) {
if(unpaddedData.get(progressTime)!=null){ if (unpaddedData.get(progressTime) != null) {
paddedResults.add(unpaddedData.get(progressTime)); paddedResults.add(unpaddedData.get(progressTime));
}else{ } else {
Info info = new Info(progressTime, jsonObject); Info info = new Info(progressTime, jsonObject);
paddedResults.add(info); paddedResults.add(info);
} }
} }
return paddedResults; return paddedResults;
} }
/** /**
* Return results with padding if pad is set to true. * Return results with padding if pad is set to true.
* @param usageRecordType the UsageRecord type to query *
* @param temporalConstraint the temporal interval and the granularity * @param aggregatedRecordClass
* @param filters the list keys to filter (in AND) * the UsageRecord type to query
* @param pad indicate is the results have to be padded with zeros when * @param temporalConstraint
* there is no data available at certain data points of sequence * the temporal interval and the granularity
* @return the requested list of Info * @param filters
* @throws Exception if fails * the list keys to filter (in AND)
* @param pad
* indicate is the results have to be padded with zeros when
* there is no data available at certain data points of sequence
* @return the requested list of Info
* @throws Exception
* if fails
*/ */
public List<Info> getInfo(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType, public List<Info> getInfo(
TemporalConstraint temporalConstraint, List<Filter> filters, boolean pad) throws Exception { Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
Map<Calendar, Info> unpaddedResults = accountingPersistenceQuery.query(usageRecordType, temporalConstraint, filters); TemporalConstraint temporalConstraint, List<Filter> filters,
if(!pad){ boolean pad) throws Exception {
Map<Calendar, Info> unpaddedResults = accountingPersistenceQuery
.getTimeSeries(aggregatedRecordClass, temporalConstraint,
filters);
if (!pad) {
return new ArrayList<Info>(unpaddedResults.values()); return new ArrayList<Info>(unpaddedResults.values());
} }
return getPaddedResults(unpaddedResults, temporalConstraint); return getPaddedResults(unpaddedResults, temporalConstraint);
} }
/** /**
* Return unpadded results * Return unpadded results
* @param usageRecordType the UsageRecord type to query *
* @param temporalConstraint the temporal interval and the granularity * @param aggregatedRecordClass
* @param filters the list keys to filter (in AND) * the UsageRecord type to query
* @return the requested list of Info * @param temporalConstraint
* @throws Exception if fails * the temporal interval and the granularity
* @param filters
* the list keys to filter (in AND)
* @return the requested list of Info
* @throws Exception
* if fails
*/ */
public List<Info> getInfo(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType, public List<Info> getInfo(
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{ Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
return getInfo(usageRecordType, temporalConstraint, filters, false); TemporalConstraint temporalConstraint, List<Filter> filters)
throws Exception {
return getInfo(aggregatedRecordClass, temporalConstraint, filters,
false);
} }
/**
* Return the list of key valid for queries a certain usage record type
* @param usageRecordType the usage record type
* @return a set containing the list of key
* @throws Exception if fails
*/
public List<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType) throws Exception{
Set<String> keys = accountingPersistenceQuery.getKeys(usageRecordType);
List<String> toSort = new ArrayList<String>(keys);
Collections.sort(toSort);
return toSort;
}
public List<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType, String key) throws Exception {
Set<String> keys = accountingPersistenceQuery.getPossibleValuesForKey(usageRecordType, key);
List<String> toSort = new ArrayList<String>(keys);
Collections.sort(toSort);
return toSort;
}
} }

View File

@ -6,88 +6,98 @@ package org.gcube.accounting.analytics.persistence;
import java.util.Calendar; import java.util.Calendar;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.SortedMap;
import java.util.SortedSet;
import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.AggregatedRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
*
*/ */
public abstract class AccountingPersistenceBackendQuery { public abstract class AccountingPersistenceBackendQuery {
private static final Logger logger = LoggerFactory.getLogger(AccountingPersistenceBackendQuery.class);
public static final int KEY_VALUES_LIMIT = 25; public static final int KEY_VALUES_LIMIT = 25;
protected abstract void prepareConnection(AccountingPersistenceBackendQueryConfiguration configuration) throws Exception; protected abstract void prepareConnection(AccountingPersistenceBackendQueryConfiguration configuration) throws Exception;
protected abstract Map<Calendar, Info> reallyQuery(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception;
/** /**
* Query the persistence obtaining a Map where the date is the key and * Query the persistence obtaining a Map where the date is the key and
* the #Info is the value. The result is relative to an Usage Record Type, * the #Info is the value. The result is relative to an Usage Record Type,
* respect a TemporalConstraint and can be applied one or more filters. * respect a TemporalConstraint and can be applied one or more filters.
* @param recordClass the Usage Record Type of interest * @param aggregatedRecordClass the Record Class of interest
* @param temporalConstraint the TemporalConstraint (interval and aggregation) * @param temporalConstraint the TemporalConstraint (interval and aggregation)
* @param filters the filter for the query. If null or empty string get all * @param filters list of filter to obtain the time series. If null or
* data. The filters are evaluated in the order the are presented and are * empty list get all data for the interested Record Class with the applying
* considered in AND * temporal constraint. All Filter must have not null and not empty key and
* value.
* The filters are must be related to different keys and are in AND.
* If the list contains more than one filter with the same key an Exception
* is thrown.
* @return the Map containing for each date in the required interval the * @return the Map containing for each date in the required interval the
* requested data * requested data
* @throws Exception if fails * @throws Exception if fails
*/ */
public Map<Calendar, Info> query(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass, public abstract SortedMap<Calendar, Info> getTimeSeries(
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{ Class<? extends AggregatedRecord<?, ?>> aggregatedRecordClass,
logger.trace("Request query: RecordClass={}, {}={}, {}s={}", recordClass.newInstance().getRecordType(), TemporalConstraint temporalConstraint,
TemporalConstraint.class.getSimpleName(), temporalConstraint.toString(), List<Filter> filters) throws Exception;
Filter.class.getSimpleName(), filters);
return reallyQuery(recordClass, temporalConstraint, filters);
}
/**
* Return the list of key valid for queries a certain usage record type
* @param recordClass the usage record class
* @return a set containing the list of key
* @throws Exception if fails
*/
public abstract Set<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass) throws Exception;
/** /**
* Return the list of possible values for a key for a certain usage record * Return a SortedMap containing the TimeSeries for top values for a
* type. * certain key taking in account all Filters. The key is identified
* The result are limited to {@link #KEY_VALUES_LIMIT} value. * adding a Filter with a null value. Only one Filter with null value is
* If you want a different limit please use the * allowed otherwise an Exception is thrown.
* {@link #getPossibleValuesForKey(Class, String, int)} function. * The values are ordered from the most occurred value.
* Invoking this function has the same effect of invoking * @param aggregatedRecordClass the Usage Record Class of interest
* {@link #getPossibleValuesForKey(Class, String, int)} function passing * @param temporalConstraint the TemporalConstraint (interval and aggregation)
* {@link #KEY_VALUES_LIMIT} has third argument. * @param filters list of filter to obtain the time series. If null or
* @param recordClass the usage record type * empty list get all data for the interested Record Class with the applying
* @param key the key * temporal constraint. All Filter (except one) must have not null and not
* empty key and value. One Filter must have not null and not
* empty key and a null value.
* The filters are must be related to different keys and are in AND.
* If the list contains more than one filter with the same key an Exception
* is thrown.
* If the list contains more than one filter with null value an Exception
* is thrown.
* @return a set containing the list of possible values * @return a set containing the list of possible values
* @throws Exception if fails * @throws Exception if fails
*/ */
public abstract Set<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass, String key) throws Exception; public abstract SortedMap<Filter, Map<Calendar, Info>> getTopValues(
Class<? extends AggregatedRecord<?, ?>> recordClass,
TemporalConstraint temporalConstraint, List<Filter> filters)
throws Exception;
/** /**
* Return the list of possible values for a key for a certain usage record * Return the list of possible values for a key for a certain usageRecord
* type. * taking in account all Filters. The value for a certain key is identified
* The result are limited to limit value. When limit is <= 0 this means * adding a Filter with a null value. Only one Filter with null value is
* no limit. * allowed otherwise an Exception is thrown.
* @param recordClass the usage record type * The values are ordered from the most occurred value.
* @param key the key * @param aggregatedRecordClass the Usage Record Class of interest
* @param limit limit of result to return. * @param temporalConstraint the TemporalConstraint (interval and aggregation)
* @param filters list of filter to obtain the time series. If null or
* empty list get all data for the interested Record Class with the applying
* temporal constraint. All Filter (except one) must have not null and not
* empty key and value. One Filter must have not null and not
* empty key and a null value.
* The filters are must be related to different keys and are in AND.
* If the list contains more than one filter with the same key an Exception
* is thrown.
* If the list contains more than one filter with null value an Exception
* is thrown.
* @return a set containing the list of possible values * @return a set containing the list of possible values
* @throws Exception if fails * @throws Exception if fails
*/ */
public abstract Set<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass, String key, int limit) throws Exception; public abstract SortedSet<Filter> getNextPossibleValues(
Class<? extends AggregatedRecord<?, ?>> aggregatedRecordClass,
TemporalConstraint temporalConstraint,
List<Filter> filters) throws Exception;
/** /**
* Close the connection to persistence * Close the connection to persistence

View File

@ -6,78 +6,94 @@ package org.gcube.accounting.analytics.persistence;
import java.util.Calendar; import java.util.Calendar;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeSet;
import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.datamodel.UsageRecord;
import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record;
/** /**
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
* *
*/ */
public class AccountingPersistenceQuery { public class AccountingPersistenceQuery {
private static final AccountingPersistenceQuery accountingPersistenceQuery; private static final AccountingPersistenceQuery accountingPersistenceQuery;
private AccountingPersistenceQuery(){} private AccountingPersistenceQuery() {
}
static { static {
accountingPersistenceQuery = new AccountingPersistenceQuery(); accountingPersistenceQuery = new AccountingPersistenceQuery();
} }
protected static synchronized AccountingPersistenceQuery getInstance(){ protected static synchronized AccountingPersistenceQuery getInstance() {
return accountingPersistenceQuery; return accountingPersistenceQuery;
} }
public static SortedSet<String> getQuerableKeys(
/** @SuppressWarnings("rawtypes") AggregatedRecord instance)
* Query the persistence obtaining a Map where the date is the key and throws Exception {
* the #Info is the value. The result is relative to an Usage Record Type, SortedSet<String> properties = new TreeSet<>(
* respect a TemporalConstraint and can be applied one or more filters. instance.getRequiredFields());
* @param recordClass the Usage Record Type of interest
* @param temporalConstraint the TemporalConstraint (interval and aggregation) properties.removeAll(instance.getAggregatedFields());
* @param filters the filter for the query. If null or empty string get all properties.removeAll(instance.getComputedFields());
* data. The filters are evaluated in the order the are presented and are properties.remove(Record.ID);
* considered in AND properties.remove(Record.CREATION_TIME);
* @return the Map containing for each date in the required interval the properties.remove(Record.RECORD_TYPE);
* requested data properties.remove(UsageRecord.SCOPE);
* @throws Exception if fails
*/ return properties;
public Map<Calendar, Info> query(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{
return AccountingPersistenceBackendQueryFactory.getInstance().query(recordClass, temporalConstraint, filters);
} }
/** public static SortedSet<String> getQuerableKeys(
* Return the list of key valid for queries a certain usage record type Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass)
* @param recordClass the usage record type throws Exception {
* @return a set containing the list of key AggregatedRecord<?,?> instance = aggregatedRecordClass.newInstance();
* @throws Exception if fails return getQuerableKeys(instance);
*/
public Set<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass) throws Exception {
return AccountingPersistenceBackendQueryFactory.getInstance().getKeys(recordClass);
} }
public Map<Calendar, Info> getTimeSeries(
/** Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
* Return the list of possible values for a key for a certain usage record type TemporalConstraint temporalConstraint, List<Filter> filters)
* @param usageRecordType the usage record type throws Exception {
* @param key the key return AccountingPersistenceBackendQueryFactory.getInstance()
* @return a set containing the list of possible values .getTimeSeries(aggregatedRecordClass, temporalConstraint,
* @throws Exception if fails filters);
*/
public Set<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType, String key) throws Exception {
return AccountingPersistenceBackendQueryFactory.getInstance().getPossibleValuesForKey(usageRecordType, key);
} }
public static SortedMap<Filter, Map<Calendar, Info>> getTopValues(
Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
TemporalConstraint temporalConstraint, List<Filter> filters)
throws Exception {
return AccountingPersistenceBackendQueryFactory.getInstance()
.getTopValues(aggregatedRecordClass, temporalConstraint,
filters);
}
public static SortedSet<Filter> getNextPossibleValues(
Class<? extends AggregatedRecord<?,?>> aggregatedRecordClass,
TemporalConstraint temporalConstraint, List<Filter> filters)
throws Exception {
return AccountingPersistenceBackendQueryFactory.getInstance()
.getNextPossibleValues(aggregatedRecordClass,
temporalConstraint, filters);
}
/** /**
* Close the connection to persistence * Close the connection to persistence
* @throws Exception if the close fails *
* @throws Exception
* if the close fails
*/ */
public void close() throws Exception { public void close() throws Exception {
AccountingPersistenceBackendQueryFactory.getInstance().close(); AccountingPersistenceBackendQueryFactory.getInstance().close();
} }
} }