Compare commits
34 Commits
Author | SHA1 | Date |
---|---|---|
Luca Frosini | c607c36eff | |
Luca Frosini | 265d1ca1d6 | |
luca.frosini | 2d9eb7986f | |
Luca Frosini | 6e0d1c81e3 | |
Luca Frosini | 3ff8c988f9 | |
Luca Frosini | 53aa3bafce | |
Luca Frosini | e56123636f | |
Luca Frosini | a6237c2456 | |
Luca Frosini | 08f3175380 | |
Luca Frosini | a39ffc0eab | |
Luca Frosini | 186f52cdb5 | |
Luca Frosini | f0b2c8d61f | |
Luca Frosini | 72a4bb68b1 | |
Luca Frosini | eb2c3ba723 | |
Luca Frosini | 0c2345f27d | |
Luca Frosini | d95585c823 | |
Luca Frosini | 24e092c52f | |
Luca Frosini | 6752b45644 | |
Luca Frosini | 3c7f8d9de7 | |
Luca Frosini | c70970cf24 | |
Luca Frosini | bbd53c2005 | |
Luca Frosini | 1482e8cfea | |
Luca Frosini | 77b5290e53 | |
Luca Frosini | 6aa36d7ad2 | |
Luca Frosini | a113390933 | |
Luca Frosini | 3657057e48 | |
Luca Frosini | ca5ddaa619 | |
Luca Frosini | 8e0b2e9e88 | |
Luca Frosini | 252541254d | |
Luca Frosini | b9c3df0f7a | |
Luca Frosini | 16da114f8b | |
Luca Frosini | 7fb9fa2523 | |
Luca Frosini | 05fd467232 | |
Luca Frosini | 88737307f0 |
|
@ -26,3 +26,4 @@ hs_err_pid*
|
|||
/target/
|
||||
/.classpath
|
||||
/.project
|
||||
/.DS_Store
|
||||
|
|
16
CHANGELOG.md
16
CHANGELOG.md
|
@ -2,7 +2,21 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
|||
|
||||
# Changelog for Accounting Analytics Backend Connector for PostgreSQL
|
||||
|
||||
## [v2.0.1]
|
||||
|
||||
## [v1.0.0]
|
||||
- Enhanced accounting-postgresql-utilities range
|
||||
|
||||
|
||||
## [v2.0.0]
|
||||
|
||||
- Added possibility of having multiple values for the same key which will be used in OR [#21353]
|
||||
|
||||
|
||||
## [v1.0.1]
|
||||
|
||||
- Fixed bug on query
|
||||
|
||||
|
||||
## [v1.0.0] [r5.2.0] - 2021-05-04
|
||||
|
||||
- First Release
|
||||
|
|
8
pom.xml
8
pom.xml
|
@ -9,7 +9,7 @@
|
|||
</parent>
|
||||
<groupId>org.gcube.accounting</groupId>
|
||||
<artifactId>accounting-analytics-persistence-postgresql</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<version>2.0.1</version>
|
||||
<name>Accounting Analytics Backend Connector for PostgreSQL</name>
|
||||
<description>Accounting Analytics Backend Connector for PostgreSQL</description>
|
||||
|
||||
|
@ -44,12 +44,12 @@
|
|||
<dependency>
|
||||
<groupId>org.gcube.accounting</groupId>
|
||||
<artifactId>accounting-analytics</artifactId>
|
||||
<version>[3.0.0, 4.0.0-SNAPSHOT)</version>
|
||||
<version>[4.0.0, 5.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.accounting</groupId>
|
||||
<artifactId>accounting-postgresql-utilities</artifactId>
|
||||
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
|
||||
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
|
@ -88,4 +88,4 @@
|
|||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
</project>
|
||||
|
|
|
@ -9,6 +9,7 @@ import java.sql.ResultSet;
|
|||
import java.sql.Statement;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -23,23 +24,20 @@ import org.gcube.accounting.analytics.Info;
|
|||
import org.gcube.accounting.analytics.NumberedFilter;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||
import org.gcube.accounting.analytics.UsageValue;
|
||||
import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException;
|
||||
import org.gcube.accounting.analytics.exception.KeyException;
|
||||
import org.gcube.accounting.analytics.exception.ValueException;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||
import org.gcube.accounting.persistence.AccountingPersistenceConfiguration;
|
||||
import org.gcube.accounting.utility.postgresql.RecordToDBConnection;
|
||||
import org.gcube.accounting.utility.postgresql.RecordToDBFields;
|
||||
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
|
||||
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.gcube.documentstore.records.AggregatedRecord;
|
||||
import org.gcube.documentstore.records.Record;
|
||||
import org.gcube.documentstore.records.RecordUtility;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
@ -50,27 +48,31 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
|
||||
protected static final Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryPostgreSQL.class);
|
||||
|
||||
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z";
|
||||
|
||||
|
||||
public static final String URL_PROPERTY_KEY = AccountingPersistenceConfiguration.URL_PROPERTY_KEY;
|
||||
// private String baseURL;
|
||||
|
||||
public static final int DEFAULT_TOP_LIMIT = 10;
|
||||
|
||||
protected AccountingPersistenceBackendQueryConfiguration configuration;
|
||||
|
||||
protected ObjectMapper objectMapper;
|
||||
|
||||
protected Class<? extends AggregatedRecord<?, ?>> clz;
|
||||
protected TemporalConstraint temporalConstraint;
|
||||
|
||||
protected Set<String> contexts;
|
||||
protected Collection<Filter> filters;
|
||||
|
||||
protected RecordToDBMapping recordToDBMapping;
|
||||
|
||||
static {
|
||||
// One Record per package is enough
|
||||
RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage());
|
||||
RecordUtility.addRecordPackage(AggregatedServiceUsageRecord.class.getPackage());
|
||||
}
|
||||
|
||||
protected Connection getConnection(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
|
||||
RecordToDBConnection recordDBInfo = RecordToDBMapping.getRecordDBInfo(clz);
|
||||
if(recordDBInfo == null) {
|
||||
RecordToDBMapping.addRecordToDB(clz, configuration);
|
||||
recordDBInfo = RecordToDBMapping.getRecordDBInfo(clz);
|
||||
}
|
||||
return recordDBInfo.getConnection();
|
||||
public AccountingPersistenceQueryPostgreSQL() {
|
||||
this.objectMapper = new ObjectMapper();
|
||||
this.recordToDBMapping = new RecordToDBMapping();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -80,16 +82,65 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
for(String typeName : aggregatedRecords.keySet()) {
|
||||
try {
|
||||
Class<? extends AggregatedRecord<?,?>> clz = aggregatedRecords.get(typeName);
|
||||
RecordToDBMapping.addRecordToDB(clz, configuration);
|
||||
recordToDBMapping.addRecordToDB(clz, configuration);
|
||||
} catch (Exception e) {
|
||||
new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected SortedMap<Calendar, Info> getTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters, Set<String> contexts)
|
||||
throws Exception {
|
||||
protected Connection getConnection(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
|
||||
RecordToDBConnection recordDBConnection = recordToDBMapping.getRecordDBInfo(clz);
|
||||
if(recordDBConnection == null) {
|
||||
recordToDBMapping.addRecordToDB(clz, configuration);
|
||||
recordDBConnection = recordToDBMapping.getRecordDBInfo(clz);
|
||||
}
|
||||
return recordDBConnection.getConnection();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setRequestedRecords(Class<? extends AggregatedRecord<?, ?>> clz) {
|
||||
this.clz = clz;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTemporalConstraint(TemporalConstraint temporalConstraint) {
|
||||
this.temporalConstraint = temporalConstraint;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setContexts(Set<String> contexts) {
|
||||
this.contexts = contexts;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public void setFilters(Collection<? extends Filter> filters) {
|
||||
this.filters = (Collection<Filter>) filters;
|
||||
}
|
||||
|
||||
public RecordToDBMapping getRecordToDBMapping() {
|
||||
return recordToDBMapping;
|
||||
}
|
||||
|
||||
protected void addProperty(ObjectNode objectNode, String key, Object value) {
|
||||
if(value instanceof Number) {
|
||||
|
||||
if(value instanceof Integer) {
|
||||
objectNode.put(key, (int) value);
|
||||
return;
|
||||
}
|
||||
|
||||
Long longValue = Long.valueOf(value.toString());
|
||||
objectNode.put(key, longValue);
|
||||
return;
|
||||
}
|
||||
|
||||
objectNode.put(key, (String) value.toString());
|
||||
|
||||
}
|
||||
|
||||
protected SortedMap<Calendar, Info> getTimeSeries(Set<String> contexts) throws Exception {
|
||||
Connection connection = getConnection(clz);
|
||||
try {
|
||||
Statement statement = connection.createStatement();
|
||||
|
@ -104,7 +155,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
String sql = query.getTimeSeriesQuery();
|
||||
|
||||
Set<String> requestedTableField = query.getRequestedTableField();
|
||||
RecordToDBFields recordToDBMapper = query.getRecordToDBMapper();
|
||||
RecordToDBFields recordToDBMapper = query.getRecordToDBFields();
|
||||
|
||||
logger.trace("Going to request the following query: {}", sql);
|
||||
ResultSet resultSet = statement.executeQuery(sql);
|
||||
|
@ -114,15 +165,15 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
OffsetDateTime offsetDateTime = resultSet.getObject(tableFieldName, OffsetDateTime.class);
|
||||
Calendar calendar = getCalendar(offsetDateTime);
|
||||
|
||||
JSONObject jsonObject = new JSONObject();
|
||||
ObjectNode objectNode = objectMapper.createObjectNode();
|
||||
|
||||
for(String tableField : requestedTableField) {
|
||||
String usageRecordField = recordToDBMapper.getRecordField(tableField);
|
||||
Object object = resultSet.getObject(tableField);
|
||||
jsonObject.put(usageRecordField, object);
|
||||
Object obj = resultSet.getObject(tableField);
|
||||
addProperty(objectNode, usageRecordField, obj);
|
||||
}
|
||||
|
||||
Info info = new Info(calendar, jsonObject);
|
||||
Info info = new Info(calendar, objectNode);
|
||||
result.put(calendar, info);
|
||||
}
|
||||
|
||||
|
@ -133,13 +184,8 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<Calendar, Info> getTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters)
|
||||
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
|
||||
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
|
||||
Set<String> contexts = new HashSet<>();
|
||||
contexts.add(context);
|
||||
return getTimeSeries(clz, temporalConstraint, filters, contexts);
|
||||
public SortedMap<Calendar, Info> getTimeSeries() throws Exception {
|
||||
return getTimeSeries(contexts);
|
||||
}
|
||||
|
||||
protected Calendar getCalendar(OffsetDateTime offsetDateTime) {
|
||||
|
@ -148,25 +194,17 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
calendar.setTimeInMillis(epochMillis);
|
||||
return calendar;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<Calendar, Info> getNoContextTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters)
|
||||
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
|
||||
return getTimeSeries(clz, temporalConstraint, filters, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<Filter, SortedMap<Calendar, Info>> getContextTimeSeries(
|
||||
Class<? extends AggregatedRecord<?, ?>> clz, TemporalConstraint temporalConstraint, List<Filter> filters,
|
||||
List<String> contexts) throws Exception {
|
||||
public SortedMap<Filter, SortedMap<Calendar, Info>> getContextTimeSeries() throws Exception {
|
||||
|
||||
SortedMap<Filter,SortedMap<Calendar,Info>> ret = new TreeMap<>();
|
||||
for(String context : contexts) {
|
||||
Filter contextFilter = new Filter("context", context);
|
||||
Set<String> timeSeriesContexts = new HashSet<>();
|
||||
timeSeriesContexts.add(context);
|
||||
SortedMap<Calendar, Info> timeSeries = getTimeSeries(clz, temporalConstraint, filters, timeSeriesContexts);
|
||||
Filter contextFilter = new Filter("scope", context);
|
||||
Set<String> ctxs = new HashSet<>();
|
||||
ctxs.add(context);
|
||||
|
||||
SortedMap<Calendar, Info> timeSeries = getTimeSeries(ctxs);
|
||||
if(!timeSeries.isEmpty()) {
|
||||
ret.put(contextFilter, timeSeries);
|
||||
}
|
||||
|
@ -174,9 +212,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
return ret;
|
||||
}
|
||||
|
||||
protected SortedSet<NumberedFilter> getNumberedValues(Class<? extends AggregatedRecord<?, ?>> clz,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters, String key,
|
||||
String orderingProperty, Integer limit) throws Exception {
|
||||
protected SortedSet<NumberedFilter> getNumberedValues(String key, String orderingProperty, Integer limit) throws Exception {
|
||||
Connection connection = getConnection(clz);
|
||||
try {
|
||||
Statement statement = connection.createStatement();
|
||||
|
@ -185,16 +221,12 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(clz);
|
||||
}
|
||||
|
||||
if(limit == null) {
|
||||
limit = 50;
|
||||
if(limit!= null && limit<1) {
|
||||
limit = DEFAULT_TOP_LIMIT;
|
||||
}
|
||||
|
||||
SortedSet<NumberedFilter> result = new TreeSet<>();
|
||||
|
||||
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
|
||||
Set<String> contexts = new HashSet<>();
|
||||
contexts.add(context);
|
||||
|
||||
Query query = new Query(clz);
|
||||
query.setTemporalConstraint(temporalConstraint);
|
||||
query.setFilters(filters);
|
||||
|
@ -206,15 +238,15 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
String sql = query.getNextPossibleValueQuery();
|
||||
|
||||
// List<String> requestedTableField = query.getRequestedTableField();
|
||||
RecordToDBFields recordToDBMapper = query.getRecordToDBMapper();
|
||||
RecordToDBFields recordToDBFields = query.getRecordToDBFields();
|
||||
|
||||
logger.trace("Going to request the following query: {}", sql);
|
||||
ResultSet resultSet = statement.executeQuery(sql);
|
||||
|
||||
String tableFieldORderingProperty = recordToDBMapper.getTableField(orderingProperty);
|
||||
String tableFieldORderingProperty = recordToDBFields.getTableField(orderingProperty);
|
||||
|
||||
while (resultSet.next()) {
|
||||
String tableFieldKey = recordToDBMapper.getTableField(key);
|
||||
String tableFieldKey = recordToDBFields.getTableField(key);
|
||||
Object value = resultSet.getObject(tableFieldKey);
|
||||
|
||||
Object numberObject = resultSet.getObject(tableFieldORderingProperty);
|
||||
|
@ -229,34 +261,26 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
}
|
||||
|
||||
@Override
|
||||
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters, String key) throws Exception {
|
||||
return getNumberedValues(clz, temporalConstraint, filters, key, null, null);
|
||||
public SortedSet<NumberedFilter> getFilterValues(String key) throws Exception {
|
||||
return getNumberedValues(key, null, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters, String key, Integer limit) throws Exception {
|
||||
return getNumberedValues(clz, temporalConstraint, filters, key, null, limit);
|
||||
public SortedSet<NumberedFilter> getFilterValues(String key, Integer limit) throws Exception {
|
||||
return getNumberedValues(key, null, limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues(
|
||||
Class<? extends AggregatedRecord<?, ?>> clz, TemporalConstraint temporalConstraint, List<Filter> filters,
|
||||
String topKey, String orderingProperty)
|
||||
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
|
||||
|
||||
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
|
||||
Set<String> contexts = new HashSet<>();
|
||||
contexts.add(context);
|
||||
public SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues(String topKey, String orderingProperty, Integer limit)
|
||||
throws Exception {
|
||||
|
||||
SortedMap<NumberedFilter,SortedMap<Calendar,Info>> ret = new TreeMap<>();
|
||||
|
||||
SortedSet<NumberedFilter> top = getNumberedValues(clz, temporalConstraint, filters, topKey, orderingProperty, 10);
|
||||
SortedSet<NumberedFilter> top = getNumberedValues(topKey, orderingProperty, limit);
|
||||
|
||||
for(NumberedFilter numberedFilter : top) {
|
||||
filters.add(numberedFilter);
|
||||
SortedMap<Calendar,Info> map = getTimeSeries(clz, temporalConstraint, filters, contexts);
|
||||
SortedMap<Calendar,Info> map = getTimeSeries();
|
||||
ret.put(numberedFilter, map);
|
||||
filters.remove(numberedFilter);
|
||||
}
|
||||
|
@ -275,7 +299,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
query.setRecordId(recordId);
|
||||
String sql = query.getRecordQuery();
|
||||
|
||||
RecordToDBFields recordToDBMapper = query.getRecordToDBMapper();
|
||||
RecordToDBFields recordToDBFields = query.getRecordToDBFields();
|
||||
|
||||
ResultSet resultSet = statement.executeQuery(sql);
|
||||
resultSet.next();
|
||||
|
@ -284,7 +308,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
Set<String> requiredFields = instance.getRequiredFields();
|
||||
|
||||
for(String recordField : requiredFields) {
|
||||
String tableField = recordToDBMapper.getTableField(recordField);
|
||||
String tableField = recordToDBFields.getTableField(recordField);
|
||||
Serializable serializable;
|
||||
switch (recordField) {
|
||||
case AggregatedRecord.START_TIME: case AggregatedRecord.END_TIME: case AggregatedRecord.CREATION_TIME:
|
||||
|
@ -309,42 +333,32 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
|
|||
}
|
||||
|
||||
@Override
|
||||
public SortedSet<String> getSpaceProvidersIds() throws Exception {
|
||||
Class<? extends AggregatedRecord<?, ?>> aggregatedRecordClass = AggregatedStorageStatusRecord.class;
|
||||
Connection connection = getConnection(aggregatedRecordClass);
|
||||
try {
|
||||
Statement statement = connection.createStatement();
|
||||
|
||||
Query query = new Query(aggregatedRecordClass);
|
||||
query.setTableFieldToRequest(AggregatedStorageStatusRecord.PROVIDER_ID);
|
||||
String sql = query.getDinstinctValuesQuery();
|
||||
|
||||
SortedSet<String> providersIds = new TreeSet<>();
|
||||
|
||||
logger.trace("Going to request the following query: {}", sql);
|
||||
ResultSet resultSet = statement.executeQuery(sql);
|
||||
|
||||
while (resultSet.next()) {
|
||||
String id = resultSet.getString(1);
|
||||
providersIds.add(id);
|
||||
public SortedMap<Filter, SortedMap<Calendar, Info>> getSpaceTimeSeries(Set<String> dataTypes) throws Exception {
|
||||
/*
|
||||
SortedMap<Filter, SortedMap<Calendar, Info>> sortedMap = new TreeMap<>();
|
||||
setRequestedRecords(AggregatedStorageStatusRecord.class);
|
||||
for(String dataType : dataTypes) {
|
||||
Filter filter = new Filter(StorageStatusRecord.DATA_TYPE, dataType);
|
||||
if(filters == null) {
|
||||
filters = new HashSet<>();
|
||||
}
|
||||
filters.add(filter);
|
||||
|
||||
return providersIds;
|
||||
}finally {
|
||||
connection.close();
|
||||
}
|
||||
SortedMap<Calendar, Info> timeSeries = getTimeSeries();
|
||||
sortedMap.put(filter, timeSeries);
|
||||
|
||||
filters.remove(filter);
|
||||
}
|
||||
return sortedMap;
|
||||
*/
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<UsageValue> getUsageValueQuotaTotal(List<UsageValue> listUsage) throws Exception {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<Filter, SortedMap<Calendar, Long>> getSpaceTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
|
||||
TemporalConstraint temporalConstraint, List<Filter> filters, List<String> providersId) throws Exception {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
|
|
|
@ -2,10 +2,12 @@ package org.gcube.accounting.analytics.persistence.postgresql;
|
|||
|
||||
import java.sql.SQLException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.accounting.analytics.Filter;
|
||||
|
@ -14,6 +16,7 @@ import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
|||
import org.gcube.accounting.analytics.TemporalConstraint.CalendarEnum;
|
||||
import org.gcube.accounting.datamodel.UsageRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
|
||||
import org.gcube.accounting.utility.postgresql.PostgreSQLQuery;
|
||||
import org.gcube.accounting.utility.postgresql.RecordToDBFields;
|
||||
|
@ -32,7 +35,7 @@ public class Query extends PostgreSQLQuery {
|
|||
private Set<String> requestedTableField;
|
||||
|
||||
protected TemporalConstraint temporalConstraint;
|
||||
protected List<Filter> filters;
|
||||
protected Map<String, Collection<String>> filters;
|
||||
protected Set<String> contexts;
|
||||
|
||||
private String tableFieldToRequest;
|
||||
|
@ -41,25 +44,8 @@ public class Query extends PostgreSQLQuery {
|
|||
|
||||
private String recordId;
|
||||
|
||||
protected String orConditionKey;
|
||||
protected Set<String> orConditionValues;
|
||||
|
||||
public void setOrConditionKey(String orConditionKey) {
|
||||
this.orConditionKey = orConditionKey;
|
||||
}
|
||||
|
||||
public void setOrConditionValues(Collection<String> orConditionValues) {
|
||||
this.orConditionValues = new HashSet<>(orConditionValues);
|
||||
}
|
||||
|
||||
public void addOrConditionValue(String conditionValue) {
|
||||
if(this.orConditionValues == null) {
|
||||
this.orConditionValues = new HashSet<>();
|
||||
}
|
||||
this.orConditionValues.add(conditionValue);
|
||||
}
|
||||
|
||||
public Query(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
|
||||
super();
|
||||
this.clz = clz;
|
||||
this.recordToDBFields = RecordToDBMapping.getRecordToDBFields(clz);
|
||||
}
|
||||
|
@ -72,8 +58,20 @@ public class Query extends PostgreSQLQuery {
|
|||
this.temporalConstraint = temporalConstraint;
|
||||
}
|
||||
|
||||
public void setFilters(List<Filter> filters) {
|
||||
this.filters = filters;
|
||||
public void setFilters(Collection<? extends Filter> filters) {
|
||||
this.filters = new HashMap<>();
|
||||
if(filters!=null && filters.size()>0) {
|
||||
for(Filter filter : filters) {
|
||||
String key = filter.getKey();
|
||||
String value = filter.getValue();
|
||||
Collection<String> list = this.filters.get(key);
|
||||
if(list == null) {
|
||||
list = new ArrayList<>();
|
||||
this.filters.put(key, list);
|
||||
}
|
||||
list.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void setContexts(Set<String> contexts) {
|
||||
|
@ -103,7 +101,7 @@ public class Query extends PostgreSQLQuery {
|
|||
this.recordId = recordId;
|
||||
}
|
||||
|
||||
public RecordToDBFields getRecordToDBMapper() {
|
||||
public RecordToDBFields getRecordToDBFields() {
|
||||
return recordToDBFields;
|
||||
}
|
||||
|
||||
|
@ -123,26 +121,29 @@ public class Query extends PostgreSQLQuery {
|
|||
|
||||
protected void addFilters() throws SQLException {
|
||||
if(filters!=null && filters.size()>0) {
|
||||
// The first filter if the time_bucket
|
||||
stringBuffer.append(" AND ");
|
||||
if(filters.size()>1) {
|
||||
stringBuffer.append("(");
|
||||
}
|
||||
boolean first = true;
|
||||
for(Filter filter : filters) {
|
||||
for(String key : filters.keySet()) {
|
||||
if(first) {
|
||||
first = false;
|
||||
}else {
|
||||
stringBuffer.append(" AND ");
|
||||
}
|
||||
appendTableField(filter.getKey());
|
||||
stringBuffer.append("=");
|
||||
appendValue(filter.getValue());
|
||||
addOrConditions(key, filters.get(key));
|
||||
}
|
||||
if(filters.size()>1) {
|
||||
stringBuffer.append(")");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void addOrConditions(String key, Set<String> values) throws SQLException {
|
||||
protected void addOrConditions(String key, Collection<String> values) throws SQLException {
|
||||
if(values!=null && values.size()>0) {
|
||||
// The first filter if the time_bucket
|
||||
stringBuffer.append(" AND (");
|
||||
if(values.size()>1) {
|
||||
stringBuffer.append("(");
|
||||
}
|
||||
boolean first = true;
|
||||
for(String value : values) {
|
||||
if(first) {
|
||||
|
@ -154,19 +155,16 @@ public class Query extends PostgreSQLQuery {
|
|||
stringBuffer.append("=");
|
||||
appendValue(value);
|
||||
}
|
||||
stringBuffer.append(")");
|
||||
if(values.size()>1) {
|
||||
stringBuffer.append(")");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void addContextFilter() throws SQLException {
|
||||
addOrConditions(UsageRecord.SCOPE, contexts);
|
||||
}
|
||||
|
||||
protected void addOrConditionFilter() throws SQLException {
|
||||
addOrConditions(orConditionKey, orConditionValues);
|
||||
}
|
||||
|
||||
protected void addEmittedFields(boolean addNested) throws Exception {
|
||||
Set<String> aggregatedField = clz.newInstance().getAggregatedFields();
|
||||
for(String fieldName : aggregatedField) {
|
||||
|
@ -187,8 +185,22 @@ public class Query extends PostgreSQLQuery {
|
|||
stringBuffer.append(") AS ");
|
||||
break;
|
||||
|
||||
case AggregatedStorageStatusRecord.DATA_COUNT:
|
||||
stringBuffer.append(", MAX(");
|
||||
if(addNested) {
|
||||
stringBuffer.append(NESTED_TABLE_NAME);
|
||||
stringBuffer.append(".");
|
||||
}
|
||||
stringBuffer.append(dbField);
|
||||
stringBuffer.append(") AS ");
|
||||
break;
|
||||
|
||||
case AggregatedStorageUsageRecord.DATA_VOLUME:
|
||||
stringBuffer.append(", SUM(");
|
||||
if(clz.isAssignableFrom(AggregatedStorageUsageRecord.class)) {
|
||||
stringBuffer.append(", SUM(");
|
||||
}else if(clz.isAssignableFrom(AggregatedStorageStatusRecord.class)){
|
||||
stringBuffer.append(", MAX(");
|
||||
}
|
||||
if(addNested) {
|
||||
stringBuffer.append(NESTED_TABLE_NAME);
|
||||
stringBuffer.append(".");
|
||||
|
@ -262,7 +274,7 @@ public class Query extends PostgreSQLQuery {
|
|||
}
|
||||
|
||||
protected void addTemporalConstraintToQuery() throws SQLException {
|
||||
stringBuffer.append(" WHERE ");
|
||||
stringBuffer.append(" WHERE (");
|
||||
String tableField = getTableField(AggregatedRecord.START_TIME);
|
||||
stringBuffer.append(tableField);
|
||||
stringBuffer.append(" > ");
|
||||
|
@ -271,8 +283,9 @@ public class Query extends PostgreSQLQuery {
|
|||
stringBuffer.append(" AND ");
|
||||
tableField = getTableField(AggregatedRecord.END_TIME);
|
||||
stringBuffer.append(tableField);
|
||||
stringBuffer.append(" < ");
|
||||
stringBuffer.append(" <= ");
|
||||
appendValue(temporalConstraint.getAlignedEndTime());
|
||||
stringBuffer.append(")");
|
||||
}
|
||||
|
||||
protected void addDateGroupBy(boolean addNested) {
|
||||
|
@ -305,11 +318,9 @@ public class Query extends PostgreSQLQuery {
|
|||
}
|
||||
|
||||
protected void addLimit() {
|
||||
stringBuffer.append(" LIMIT ");
|
||||
if(limit != null) {
|
||||
stringBuffer.append(" LIMIT ");
|
||||
stringBuffer.append(limit.toString());
|
||||
}else {
|
||||
stringBuffer.append(50);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -358,8 +369,15 @@ public class Query extends PostgreSQLQuery {
|
|||
|
||||
addTemporalConstraintToQuery();
|
||||
|
||||
addFilters();
|
||||
addContextFilter();
|
||||
if(filters!=null && filters.size()>0) {
|
||||
stringBuffer.append(" AND ");
|
||||
addFilters();
|
||||
}
|
||||
|
||||
if(contexts!=null && contexts.size()>0) {
|
||||
stringBuffer.append(" AND ");
|
||||
addContextFilter();
|
||||
}
|
||||
|
||||
addDateGroupBy(false);
|
||||
addDateOrderBy(false);
|
||||
|
@ -383,10 +401,27 @@ public class Query extends PostgreSQLQuery {
|
|||
stringBuffer.append(" FROM ");
|
||||
stringBuffer.append(recordToDBFields.getTableName());
|
||||
|
||||
addTemporalConstraintToQuery();
|
||||
boolean addAnd = false;
|
||||
if(temporalConstraint!=null) {
|
||||
addTemporalConstraintToQuery();
|
||||
addAnd = true;
|
||||
}
|
||||
|
||||
addFilters();
|
||||
addContextFilter();
|
||||
if(filters!=null && filters.size()>0) {
|
||||
if(addAnd) {
|
||||
stringBuffer.append(" AND ");
|
||||
}
|
||||
addFilters();
|
||||
addAnd = true;
|
||||
}
|
||||
|
||||
if(contexts!=null && contexts.size()>0) {
|
||||
if(addAnd) {
|
||||
stringBuffer.append(" AND ");
|
||||
}
|
||||
addContextFilter();
|
||||
addAnd = false;
|
||||
}
|
||||
|
||||
addGroupAndOrderByForOrderByField();
|
||||
addLimit();
|
||||
|
|
|
@ -5,9 +5,12 @@ package org.gcube.accounting.analytics.persistence.postgresql;
|
|||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.SortedSet;
|
||||
|
||||
|
@ -21,10 +24,13 @@ import org.gcube.accounting.analytics.exception.KeyException;
|
|||
import org.gcube.accounting.analytics.exception.ValueException;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
|
||||
import org.gcube.accounting.datamodel.UsageRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
|
||||
import org.gcube.accounting.datamodel.basetypes.AbstractStorageUsageRecord.DataType;
|
||||
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
|
||||
import org.gcube.documentstore.records.AggregatedRecord;
|
||||
import org.gcube.documentstore.records.Record;
|
||||
|
@ -62,6 +68,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
|
||||
protected TemporalConstraint getTemporalConstraint() {
|
||||
Calendar startTimeCalendar = Calendar.getInstance();
|
||||
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
startTimeCalendar.set(Calendar.YEAR, 2015);
|
||||
startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY);
|
||||
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
|
||||
|
@ -69,6 +76,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
startTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
Calendar entTimeCalendar = Calendar.getInstance();
|
||||
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
entTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH);
|
||||
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
|
||||
|
@ -89,6 +97,231 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
return classes;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNoContextTimeSeriesYearly() throws Exception {
|
||||
|
||||
List<Filter> filters = getFilters();
|
||||
|
||||
AggregationMode aggregationMode = AggregationMode.YEARLY;
|
||||
|
||||
Calendar startTimeCalendar = Calendar.getInstance();
|
||||
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
startTimeCalendar.set(Calendar.YEAR, 2015);
|
||||
startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY);
|
||||
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
|
||||
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
startTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
Calendar entTimeCalendar = Calendar.getInstance();
|
||||
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
entTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
entTimeCalendar.set(Calendar.MONTH, Calendar.DECEMBER);
|
||||
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 31);
|
||||
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
entTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
|
||||
|
||||
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
|
||||
|
||||
logger.debug("----------------------------------------------------------------------");
|
||||
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
|
||||
|
||||
|
||||
temporalConstraint.setAggregationMode(aggregationMode);
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNoContextTimeSeriesMonthly() throws Exception {
|
||||
|
||||
List<Filter> filters = getFilters();
|
||||
|
||||
AggregationMode aggregationMode = AggregationMode.MONTHLY;
|
||||
|
||||
Calendar startTimeCalendar = Calendar.getInstance();
|
||||
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
startTimeCalendar.set(Calendar.YEAR, 2015);
|
||||
startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY);
|
||||
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
|
||||
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
startTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
Calendar entTimeCalendar = Calendar.getInstance();
|
||||
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
entTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
|
||||
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
|
||||
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
entTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
|
||||
|
||||
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
|
||||
|
||||
logger.debug("----------------------------------------------------------------------");
|
||||
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
|
||||
|
||||
|
||||
temporalConstraint.setAggregationMode(aggregationMode);
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNoContextTimeSeriesDaily() throws Exception {
|
||||
|
||||
List<Filter> filters = getFilters();
|
||||
|
||||
AggregationMode aggregationMode = AggregationMode.DAILY;
|
||||
|
||||
Calendar startTimeCalendar = Calendar.getInstance();
|
||||
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
startTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
startTimeCalendar.set(Calendar.MONTH, Calendar.OCTOBER);
|
||||
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
|
||||
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
startTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
Calendar entTimeCalendar = Calendar.getInstance();
|
||||
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
entTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
|
||||
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
|
||||
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
entTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
|
||||
|
||||
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
|
||||
|
||||
logger.debug("----------------------------------------------------------------------");
|
||||
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
|
||||
|
||||
|
||||
temporalConstraint.setAggregationMode(aggregationMode);
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNoContextTimeSeriesHourly() throws Exception {
|
||||
|
||||
List<Filter> filters = getFilters();
|
||||
|
||||
AggregationMode aggregationMode = AggregationMode.HOURLY;
|
||||
|
||||
Calendar startTimeCalendar = Calendar.getInstance();
|
||||
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
startTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
startTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
|
||||
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
|
||||
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 00);
|
||||
startTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
Calendar entTimeCalendar = Calendar.getInstance();
|
||||
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
entTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
|
||||
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
|
||||
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 23);
|
||||
entTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
|
||||
|
||||
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
|
||||
|
||||
logger.debug("----------------------------------------------------------------------");
|
||||
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
|
||||
|
||||
|
||||
temporalConstraint.setAggregationMode(aggregationMode);
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNoContextTimeSeriesMinutely() throws Exception {
|
||||
|
||||
List<Filter> filters = getFilters();
|
||||
|
||||
AggregationMode aggregationMode = AggregationMode.MINUTELY;
|
||||
|
||||
Calendar startTimeCalendar = Calendar.getInstance();
|
||||
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
startTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
startTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
|
||||
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
|
||||
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 17);
|
||||
startTimeCalendar.set(Calendar.MINUTE, 15);
|
||||
|
||||
Calendar entTimeCalendar = Calendar.getInstance();
|
||||
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
entTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
|
||||
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
|
||||
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 17);
|
||||
entTimeCalendar.set(Calendar.MINUTE, 15);
|
||||
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
|
||||
|
||||
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
|
||||
|
||||
logger.debug("----------------------------------------------------------------------");
|
||||
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
|
||||
|
||||
|
||||
temporalConstraint.setAggregationMode(aggregationMode);
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNoContextTimeSeries() throws Exception {
|
||||
|
||||
|
@ -102,7 +335,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
logger.debug("----------------------------------------------------------------------");
|
||||
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getNoContextTimeSeries(clz, temporalConstraint, filters);
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
|
@ -123,7 +360,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
|
||||
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(clz, temporalConstraint, filters);
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
|
@ -142,15 +383,21 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
|
||||
List<Class<? extends AggregatedRecord<?, ?>>> classes = getClassesToTest();
|
||||
|
||||
List<String> contexts = new ArrayList<>();
|
||||
Set<String> contexts = new HashSet<>();
|
||||
contexts.add("/gcube/devsec/devVRE");
|
||||
contexts.add("/gcube/devNext/NextNext");
|
||||
|
||||
for(Class<? extends AggregatedRecord<?, ?>> clz : classes) {
|
||||
logger.debug("----------------------------------------------------------------------");
|
||||
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
|
||||
|
||||
SortedMap<Filter, SortedMap<Calendar, Info>> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries(clz, temporalConstraint, filters, contexts);
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setContexts(contexts);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
|
||||
SortedMap<Filter, SortedMap<Calendar, Info>> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries();
|
||||
for(Filter f : contextTimeseries.keySet()) {
|
||||
logger.debug("{}", f);
|
||||
SortedMap<Calendar, Info> timeseries = contextTimeseries.get(f);
|
||||
|
@ -184,13 +431,31 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
|
||||
String key = keysToRequest.get(clz);
|
||||
|
||||
SortedSet<NumberedFilter> numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key);
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
|
||||
SortedSet<NumberedFilter> numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key);
|
||||
for(NumberedFilter numberedFilter : numberedFilters) {
|
||||
logger.debug("{}", numberedFilter);
|
||||
}
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key);
|
||||
for(NumberedFilter numberedFilter : numberedFilters) {
|
||||
logger.debug("{}", numberedFilter);
|
||||
}
|
||||
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(null);
|
||||
logger.debug("----Only first 3 results");
|
||||
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key, 3);
|
||||
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3);
|
||||
for(NumberedFilter numberedFilter : numberedFilters) {
|
||||
logger.debug("{}", numberedFilter);
|
||||
}
|
||||
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
logger.debug("----Only first 3 results");
|
||||
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3);
|
||||
for(NumberedFilter numberedFilter : numberedFilters) {
|
||||
logger.debug("{}", numberedFilter);
|
||||
}
|
||||
|
@ -207,7 +472,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
|
||||
TemporalConstraint temporalConstraint = getTemporalConstraint();
|
||||
|
||||
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty);
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedServiceUsageRecord.class);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(filters);
|
||||
|
||||
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty, null);
|
||||
for(NumberedFilter numberedFilter : topTimeSeries.keySet()) {
|
||||
logger.debug("{}", numberedFilter);
|
||||
SortedMap<Calendar, Info> timeseries = topTimeSeries.get(numberedFilter);
|
||||
|
@ -226,8 +495,63 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testGetSpaceProvidersIds() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
|
||||
SortedSet<String> spaceProvidersIds = accountingPersistenceQueryPostgreSQL.getSpaceProvidersIds();
|
||||
logger.debug("{}", spaceProvidersIds);
|
||||
public void storageStatusTest() throws Exception {
|
||||
accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedStorageStatusRecord.class);
|
||||
Collection<NumberedFilter> myFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedStorageStatusRecord.DATA_TYPE);
|
||||
accountingPersistenceQueryPostgreSQL.setFilters(myFilters);
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(getTemporalConstraint());
|
||||
accountingPersistenceQueryPostgreSQL.getTimeSeries();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSpaceTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
|
||||
|
||||
TemporalConstraint temporalConstraint = getTemporalConstraint();
|
||||
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
|
||||
|
||||
Set<String> dataTypes = new HashSet<>();
|
||||
dataTypes.add(DataType.STORAGE.name());
|
||||
|
||||
SortedMap<Filter, SortedMap<Calendar, Info>> getSpaceTimeSeries = accountingPersistenceQueryPostgreSQL.getSpaceTimeSeries(dataTypes);
|
||||
for(Filter filter : getSpaceTimeSeries.keySet()) {
|
||||
logger.debug("{}", filter);
|
||||
SortedMap<Calendar, Info> timeseries = getSpaceTimeSeries.get(filter);
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStorageTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
|
||||
AccountingPersistenceQuery apq = AccountingPersistenceQueryFactory.getInstance();
|
||||
|
||||
Calendar startTimeCalendar = Calendar.getInstance();
|
||||
startTimeCalendar.set(Calendar.YEAR, 2017);
|
||||
startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY);
|
||||
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
|
||||
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
startTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
Calendar entTimeCalendar = Calendar.getInstance();
|
||||
entTimeCalendar.set(Calendar.YEAR, 2021);
|
||||
entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH);
|
||||
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 22);
|
||||
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
entTimeCalendar.set(Calendar.MINUTE, 0);
|
||||
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.DAILY);
|
||||
|
||||
apq.setRequestedRecords(AggregatedStorageStatusRecord.class);
|
||||
apq.setTemporalConstraint(temporalConstraint);
|
||||
|
||||
SortedMap<Calendar, Info> timeseries = apq.getTimeSeries();
|
||||
for(Calendar c : timeseries.keySet()) {
|
||||
Info info = timeseries.get(c);
|
||||
logger.debug("{}", info);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
|
@ -13,7 +13,6 @@ import java.util.Set;
|
|||
import org.gcube.accounting.analytics.Filter;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||
import org.gcube.accounting.datamodel.UsageRecord;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
|
@ -60,6 +59,8 @@ public class QueryTest extends ContextTest {
|
|||
List<Filter> filters = new ArrayList<>();
|
||||
Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser());
|
||||
filters.add(filter);
|
||||
filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser());
|
||||
filters.add(filter);
|
||||
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY);
|
||||
Query query = new Query(AggregatedServiceUsageRecord.class);
|
||||
|
@ -69,9 +70,9 @@ public class QueryTest extends ContextTest {
|
|||
logger.debug(ret);
|
||||
|
||||
|
||||
String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery();
|
||||
Set<String> contexts = new HashSet<>();
|
||||
contexts.add(currentScope);
|
||||
contexts.add(DEFAULT_TEST_SCOPE);
|
||||
contexts.add(ALTERNATIVE_TEST_SCOPE);
|
||||
query.setContexts(contexts);
|
||||
ret = query.getTimeSeriesQuery();
|
||||
logger.debug(ret);
|
||||
|
@ -91,7 +92,7 @@ public class QueryTest extends ContextTest {
|
|||
@Test
|
||||
public void testGetRecordQuery() throws Exception {
|
||||
Query query = new Query(AggregatedServiceUsageRecord.class);
|
||||
query.setRecordId("fa573711-ceb6-44ba-9c83-bd47e0915b80");
|
||||
query.setRecordId("7c3fcb31-6909-451d-bae0-633b3bec0c21");
|
||||
String ret = query.getRecordQuery();
|
||||
logger.debug(ret);
|
||||
}
|
||||
|
|
|
@ -10,7 +10,7 @@
|
|||
|
||||
|
||||
<logger name="org.gcube" level="INFO" />
|
||||
<logger name="org.gcube.accounting.analytics.persistence.postgresql" level="TRACE" />
|
||||
<logger name="org.gcube.accounting.analytics" level="TRACE" />
|
||||
|
||||
<root level="WARN">
|
||||
<appender-ref ref="STDOUT" />
|
||||
|
|
Loading…
Reference in New Issue