Compare commits

...

34 Commits

Author SHA1 Message Date
Luca Frosini c607c36eff Removed -SNAPSHOT to release the component 2024-04-23 12:29:52 +02:00
Luca Frosini 265d1ca1d6 Removed static usage 2024-02-29 16:14:13 +01:00
luca.frosini 2d9eb7986f Ignored MacOs File 2023-06-21 11:24:28 +02:00
Luca Frosini 6e0d1c81e3 Fixed dependency range 2021-12-01 10:54:20 +01:00
Luca Frosini 3ff8c988f9 Removed -SNAPSHOT for release 2021-12-01 10:51:02 +01:00
Luca Frosini 53aa3bafce Fixed interval request 2021-11-26 17:24:33 +01:00
Luca Frosini e56123636f Fixed bug 2021-11-25 18:01:18 +01:00
Luca Frosini a6237c2456 Changed limit behaviour 2021-11-25 17:06:10 +01:00
Luca Frosini 08f3175380 limit null means unlimited 2021-11-25 16:59:33 +01:00
Luca Frosini a39ffc0eab added patch tom manage limit=0 2021-11-25 14:39:15 +01:00
Luca Frosini 186f52cdb5 added check on max limit 2021-11-25 12:30:13 +01:00
Luca Frosini f0b2c8d61f fixed limit issue on top values 2021-11-25 12:27:52 +01:00
Luca Frosini 72a4bb68b1 Remove duplicated line 2021-11-23 12:15:03 +01:00
Luca Frosini eb2c3ba723 Fixed query bug. Added test 2021-11-23 10:41:27 +01:00
Luca Frosini 0c2345f27d Added test 2021-11-22 18:23:12 +01:00
Luca Frosini d95585c823 Removed unused constant 2021-11-22 18:22:37 +01:00
Luca Frosini 24e092c52f fixing test 2021-11-22 15:55:20 +01:00
Luca Frosini 6752b45644 fixed test 2021-11-22 15:17:28 +01:00
Luca Frosini 3c7f8d9de7 Fixed query 2021-11-22 15:07:29 +01:00
Luca Frosini c70970cf24 Generalised flters 2021-11-22 14:19:12 +01:00
Luca Frosini bbd53c2005 usign collection in place of list 2021-11-22 13:37:31 +01:00
Luca Frosini 1482e8cfea Switched from set to list 2021-11-22 12:48:34 +01:00
Luca Frosini 77b5290e53 Merge branch 'feature/21353'
Conflicts:
	CHANGELOG.md
	pom.xml
	src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java
2021-11-02 11:47:56 +01:00
Luca Frosini 6aa36d7ad2 Fixed changelog 2021-11-02 11:31:26 +01:00
Luca Frosini a113390933 Validated query builder 2021-11-02 11:29:29 +01:00
Luca Frosini 3657057e48 Added the possibility of having OR conditions between filters with the
same key
2021-10-29 19:02:49 +02:00
Luca Frosini ca5ddaa619 Fixed CHANGELOG 2021-07-28 14:15:26 +02:00
Luca Frosini 8e0b2e9e88 Removed -SNAPSHOT for release 2021-07-28 13:07:43 +02:00
Luca Frosini 252541254d Added snapshot to allow jenkins to compile it 2021-07-28 13:07:19 +02:00
Luca Frosini b9c3df0f7a Fixed bug on query 2021-07-28 13:05:57 +02:00
Luca Frosini 16da114f8b backported patch 2021-07-28 13:04:27 +02:00
Luca Frosini 7fb9fa2523 Aligning with new model 2021-06-04 11:59:38 +02:00
Luca Frosini 05fd467232 Filtered context time series 2021-06-04 11:55:37 +02:00
Luca Frosini 88737307f0 Switching to new accounting-analytics APIs 2021-05-07 14:06:32 +02:00
8 changed files with 563 additions and 174 deletions

1
.gitignore vendored
View File

@ -26,3 +26,4 @@ hs_err_pid*
/target/ /target/
/.classpath /.classpath
/.project /.project
/.DS_Store

View File

@ -2,7 +2,21 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for Accounting Analytics Backend Connector for PostgreSQL # Changelog for Accounting Analytics Backend Connector for PostgreSQL
## [v2.0.1]
## [v1.0.0] - Enhanced accounting-postgresql-utilities range
## [v2.0.0]
- Added possibility of having multiple values for the same key which will be used in OR [#21353]
## [v1.0.1]
- Fixed bug on query
## [v1.0.0] [r5.2.0] - 2021-05-04
- First Release - First Release

View File

@ -9,7 +9,7 @@
</parent> </parent>
<groupId>org.gcube.accounting</groupId> <groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics-persistence-postgresql</artifactId> <artifactId>accounting-analytics-persistence-postgresql</artifactId>
<version>1.0.0</version> <version>2.0.1</version>
<name>Accounting Analytics Backend Connector for PostgreSQL</name> <name>Accounting Analytics Backend Connector for PostgreSQL</name>
<description>Accounting Analytics Backend Connector for PostgreSQL</description> <description>Accounting Analytics Backend Connector for PostgreSQL</description>
@ -44,12 +44,12 @@
<dependency> <dependency>
<groupId>org.gcube.accounting</groupId> <groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics</artifactId> <artifactId>accounting-analytics</artifactId>
<version>[3.0.0, 4.0.0-SNAPSHOT)</version> <version>[4.0.0, 5.0.0-SNAPSHOT)</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.gcube.accounting</groupId> <groupId>org.gcube.accounting</groupId>
<artifactId>accounting-postgresql-utilities</artifactId> <artifactId>accounting-postgresql-utilities</artifactId>
<version>[1.0.0, 2.0.0-SNAPSHOT)</version> <version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@ -88,4 +88,4 @@
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
</project> </project>

View File

@ -9,6 +9,7 @@ import java.sql.ResultSet;
import java.sql.Statement; import java.sql.Statement;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collection;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -23,23 +24,20 @@ import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.NumberedFilter;
import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.UsageValue; import org.gcube.accounting.analytics.UsageValue;
import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException;
import org.gcube.accounting.analytics.exception.KeyException;
import org.gcube.accounting.analytics.exception.ValueException;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; import org.gcube.accounting.persistence.AccountingPersistenceConfiguration;
import org.gcube.accounting.utility.postgresql.RecordToDBConnection; import org.gcube.accounting.utility.postgresql.RecordToDBConnection;
import org.gcube.accounting.utility.postgresql.RecordToDBFields; import org.gcube.accounting.utility.postgresql.RecordToDBFields;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping; import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.records.RecordUtility; import org.gcube.documentstore.records.RecordUtility;
import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -50,27 +48,31 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
protected static final Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryPostgreSQL.class); protected static final Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryPostgreSQL.class);
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z";
public static final String URL_PROPERTY_KEY = AccountingPersistenceConfiguration.URL_PROPERTY_KEY; public static final String URL_PROPERTY_KEY = AccountingPersistenceConfiguration.URL_PROPERTY_KEY;
// private String baseURL;
public static final int DEFAULT_TOP_LIMIT = 10;
protected AccountingPersistenceBackendQueryConfiguration configuration; protected AccountingPersistenceBackendQueryConfiguration configuration;
protected ObjectMapper objectMapper;
protected Class<? extends AggregatedRecord<?, ?>> clz;
protected TemporalConstraint temporalConstraint;
protected Set<String> contexts;
protected Collection<Filter> filters;
protected RecordToDBMapping recordToDBMapping;
static { static {
// One Record per package is enough // One Record per package is enough
RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage()); RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage());
RecordUtility.addRecordPackage(AggregatedServiceUsageRecord.class.getPackage()); RecordUtility.addRecordPackage(AggregatedServiceUsageRecord.class.getPackage());
} }
protected Connection getConnection(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception { public AccountingPersistenceQueryPostgreSQL() {
RecordToDBConnection recordDBInfo = RecordToDBMapping.getRecordDBInfo(clz); this.objectMapper = new ObjectMapper();
if(recordDBInfo == null) { this.recordToDBMapping = new RecordToDBMapping();
RecordToDBMapping.addRecordToDB(clz, configuration);
recordDBInfo = RecordToDBMapping.getRecordDBInfo(clz);
}
return recordDBInfo.getConnection();
} }
@Override @Override
@ -80,16 +82,65 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
for(String typeName : aggregatedRecords.keySet()) { for(String typeName : aggregatedRecords.keySet()) {
try { try {
Class<? extends AggregatedRecord<?,?>> clz = aggregatedRecords.get(typeName); Class<? extends AggregatedRecord<?,?>> clz = aggregatedRecords.get(typeName);
RecordToDBMapping.addRecordToDB(clz, configuration); recordToDBMapping.addRecordToDB(clz, configuration);
} catch (Exception e) { } catch (Exception e) {
new RuntimeException(e); new RuntimeException(e);
} }
} }
} }
protected SortedMap<Calendar, Info> getTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz, protected Connection getConnection(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters, Set<String> contexts) RecordToDBConnection recordDBConnection = recordToDBMapping.getRecordDBInfo(clz);
throws Exception { if(recordDBConnection == null) {
recordToDBMapping.addRecordToDB(clz, configuration);
recordDBConnection = recordToDBMapping.getRecordDBInfo(clz);
}
return recordDBConnection.getConnection();
}
@Override
public void setRequestedRecords(Class<? extends AggregatedRecord<?, ?>> clz) {
this.clz = clz;
}
@Override
public void setTemporalConstraint(TemporalConstraint temporalConstraint) {
this.temporalConstraint = temporalConstraint;
}
@Override
public void setContexts(Set<String> contexts) {
this.contexts = contexts;
}
@SuppressWarnings("unchecked")
@Override
public void setFilters(Collection<? extends Filter> filters) {
this.filters = (Collection<Filter>) filters;
}
public RecordToDBMapping getRecordToDBMapping() {
return recordToDBMapping;
}
protected void addProperty(ObjectNode objectNode, String key, Object value) {
if(value instanceof Number) {
if(value instanceof Integer) {
objectNode.put(key, (int) value);
return;
}
Long longValue = Long.valueOf(value.toString());
objectNode.put(key, longValue);
return;
}
objectNode.put(key, (String) value.toString());
}
protected SortedMap<Calendar, Info> getTimeSeries(Set<String> contexts) throws Exception {
Connection connection = getConnection(clz); Connection connection = getConnection(clz);
try { try {
Statement statement = connection.createStatement(); Statement statement = connection.createStatement();
@ -104,7 +155,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
String sql = query.getTimeSeriesQuery(); String sql = query.getTimeSeriesQuery();
Set<String> requestedTableField = query.getRequestedTableField(); Set<String> requestedTableField = query.getRequestedTableField();
RecordToDBFields recordToDBMapper = query.getRecordToDBMapper(); RecordToDBFields recordToDBMapper = query.getRecordToDBFields();
logger.trace("Going to request the following query: {}", sql); logger.trace("Going to request the following query: {}", sql);
ResultSet resultSet = statement.executeQuery(sql); ResultSet resultSet = statement.executeQuery(sql);
@ -114,15 +165,15 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
OffsetDateTime offsetDateTime = resultSet.getObject(tableFieldName, OffsetDateTime.class); OffsetDateTime offsetDateTime = resultSet.getObject(tableFieldName, OffsetDateTime.class);
Calendar calendar = getCalendar(offsetDateTime); Calendar calendar = getCalendar(offsetDateTime);
JSONObject jsonObject = new JSONObject(); ObjectNode objectNode = objectMapper.createObjectNode();
for(String tableField : requestedTableField) { for(String tableField : requestedTableField) {
String usageRecordField = recordToDBMapper.getRecordField(tableField); String usageRecordField = recordToDBMapper.getRecordField(tableField);
Object object = resultSet.getObject(tableField); Object obj = resultSet.getObject(tableField);
jsonObject.put(usageRecordField, object); addProperty(objectNode, usageRecordField, obj);
} }
Info info = new Info(calendar, jsonObject); Info info = new Info(calendar, objectNode);
result.put(calendar, info); result.put(calendar, info);
} }
@ -133,13 +184,8 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
} }
@Override @Override
public SortedMap<Calendar, Info> getTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz, public SortedMap<Calendar, Info> getTimeSeries() throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters) return getTimeSeries(contexts);
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>();
contexts.add(context);
return getTimeSeries(clz, temporalConstraint, filters, contexts);
} }
protected Calendar getCalendar(OffsetDateTime offsetDateTime) { protected Calendar getCalendar(OffsetDateTime offsetDateTime) {
@ -148,25 +194,17 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
calendar.setTimeInMillis(epochMillis); calendar.setTimeInMillis(epochMillis);
return calendar; return calendar;
} }
@Override
public SortedMap<Calendar, Info> getNoContextTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters)
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
return getTimeSeries(clz, temporalConstraint, filters, null);
}
@Override @Override
public SortedMap<Filter, SortedMap<Calendar, Info>> getContextTimeSeries( public SortedMap<Filter, SortedMap<Calendar, Info>> getContextTimeSeries() throws Exception {
Class<? extends AggregatedRecord<?, ?>> clz, TemporalConstraint temporalConstraint, List<Filter> filters,
List<String> contexts) throws Exception {
SortedMap<Filter,SortedMap<Calendar,Info>> ret = new TreeMap<>(); SortedMap<Filter,SortedMap<Calendar,Info>> ret = new TreeMap<>();
for(String context : contexts) { for(String context : contexts) {
Filter contextFilter = new Filter("context", context); Filter contextFilter = new Filter("scope", context);
Set<String> timeSeriesContexts = new HashSet<>(); Set<String> ctxs = new HashSet<>();
timeSeriesContexts.add(context); ctxs.add(context);
SortedMap<Calendar, Info> timeSeries = getTimeSeries(clz, temporalConstraint, filters, timeSeriesContexts);
SortedMap<Calendar, Info> timeSeries = getTimeSeries(ctxs);
if(!timeSeries.isEmpty()) { if(!timeSeries.isEmpty()) {
ret.put(contextFilter, timeSeries); ret.put(contextFilter, timeSeries);
} }
@ -174,9 +212,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
return ret; return ret;
} }
protected SortedSet<NumberedFilter> getNumberedValues(Class<? extends AggregatedRecord<?, ?>> clz, protected SortedSet<NumberedFilter> getNumberedValues(String key, String orderingProperty, Integer limit) throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters, String key,
String orderingProperty, Integer limit) throws Exception {
Connection connection = getConnection(clz); Connection connection = getConnection(clz);
try { try {
Statement statement = connection.createStatement(); Statement statement = connection.createStatement();
@ -185,16 +221,12 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(clz); orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(clz);
} }
if(limit == null) { if(limit!= null && limit<1) {
limit = 50; limit = DEFAULT_TOP_LIMIT;
} }
SortedSet<NumberedFilter> result = new TreeSet<>(); SortedSet<NumberedFilter> result = new TreeSet<>();
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>();
contexts.add(context);
Query query = new Query(clz); Query query = new Query(clz);
query.setTemporalConstraint(temporalConstraint); query.setTemporalConstraint(temporalConstraint);
query.setFilters(filters); query.setFilters(filters);
@ -206,15 +238,15 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
String sql = query.getNextPossibleValueQuery(); String sql = query.getNextPossibleValueQuery();
// List<String> requestedTableField = query.getRequestedTableField(); // List<String> requestedTableField = query.getRequestedTableField();
RecordToDBFields recordToDBMapper = query.getRecordToDBMapper(); RecordToDBFields recordToDBFields = query.getRecordToDBFields();
logger.trace("Going to request the following query: {}", sql); logger.trace("Going to request the following query: {}", sql);
ResultSet resultSet = statement.executeQuery(sql); ResultSet resultSet = statement.executeQuery(sql);
String tableFieldORderingProperty = recordToDBMapper.getTableField(orderingProperty); String tableFieldORderingProperty = recordToDBFields.getTableField(orderingProperty);
while (resultSet.next()) { while (resultSet.next()) {
String tableFieldKey = recordToDBMapper.getTableField(key); String tableFieldKey = recordToDBFields.getTableField(key);
Object value = resultSet.getObject(tableFieldKey); Object value = resultSet.getObject(tableFieldKey);
Object numberObject = resultSet.getObject(tableFieldORderingProperty); Object numberObject = resultSet.getObject(tableFieldORderingProperty);
@ -229,34 +261,26 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
} }
@Override @Override
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz, public SortedSet<NumberedFilter> getFilterValues(String key) throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters, String key) throws Exception { return getNumberedValues(key, null, null);
return getNumberedValues(clz, temporalConstraint, filters, key, null, null);
} }
@Override @Override
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz, public SortedSet<NumberedFilter> getFilterValues(String key, Integer limit) throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters, String key, Integer limit) throws Exception { return getNumberedValues(key, null, limit);
return getNumberedValues(clz, temporalConstraint, filters, key, null, limit);
} }
@Override @Override
public SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues( public SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues(String topKey, String orderingProperty, Integer limit)
Class<? extends AggregatedRecord<?, ?>> clz, TemporalConstraint temporalConstraint, List<Filter> filters, throws Exception {
String topKey, String orderingProperty)
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>();
contexts.add(context);
SortedMap<NumberedFilter,SortedMap<Calendar,Info>> ret = new TreeMap<>(); SortedMap<NumberedFilter,SortedMap<Calendar,Info>> ret = new TreeMap<>();
SortedSet<NumberedFilter> top = getNumberedValues(clz, temporalConstraint, filters, topKey, orderingProperty, 10); SortedSet<NumberedFilter> top = getNumberedValues(topKey, orderingProperty, limit);
for(NumberedFilter numberedFilter : top) { for(NumberedFilter numberedFilter : top) {
filters.add(numberedFilter); filters.add(numberedFilter);
SortedMap<Calendar,Info> map = getTimeSeries(clz, temporalConstraint, filters, contexts); SortedMap<Calendar,Info> map = getTimeSeries();
ret.put(numberedFilter, map); ret.put(numberedFilter, map);
filters.remove(numberedFilter); filters.remove(numberedFilter);
} }
@ -275,7 +299,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
query.setRecordId(recordId); query.setRecordId(recordId);
String sql = query.getRecordQuery(); String sql = query.getRecordQuery();
RecordToDBFields recordToDBMapper = query.getRecordToDBMapper(); RecordToDBFields recordToDBFields = query.getRecordToDBFields();
ResultSet resultSet = statement.executeQuery(sql); ResultSet resultSet = statement.executeQuery(sql);
resultSet.next(); resultSet.next();
@ -284,7 +308,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
Set<String> requiredFields = instance.getRequiredFields(); Set<String> requiredFields = instance.getRequiredFields();
for(String recordField : requiredFields) { for(String recordField : requiredFields) {
String tableField = recordToDBMapper.getTableField(recordField); String tableField = recordToDBFields.getTableField(recordField);
Serializable serializable; Serializable serializable;
switch (recordField) { switch (recordField) {
case AggregatedRecord.START_TIME: case AggregatedRecord.END_TIME: case AggregatedRecord.CREATION_TIME: case AggregatedRecord.START_TIME: case AggregatedRecord.END_TIME: case AggregatedRecord.CREATION_TIME:
@ -309,42 +333,32 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
} }
@Override @Override
public SortedSet<String> getSpaceProvidersIds() throws Exception { public SortedMap<Filter, SortedMap<Calendar, Info>> getSpaceTimeSeries(Set<String> dataTypes) throws Exception {
Class<? extends AggregatedRecord<?, ?>> aggregatedRecordClass = AggregatedStorageStatusRecord.class; /*
Connection connection = getConnection(aggregatedRecordClass); SortedMap<Filter, SortedMap<Calendar, Info>> sortedMap = new TreeMap<>();
try { setRequestedRecords(AggregatedStorageStatusRecord.class);
Statement statement = connection.createStatement(); for(String dataType : dataTypes) {
Filter filter = new Filter(StorageStatusRecord.DATA_TYPE, dataType);
Query query = new Query(aggregatedRecordClass); if(filters == null) {
query.setTableFieldToRequest(AggregatedStorageStatusRecord.PROVIDER_ID); filters = new HashSet<>();
String sql = query.getDinstinctValuesQuery();
SortedSet<String> providersIds = new TreeSet<>();
logger.trace("Going to request the following query: {}", sql);
ResultSet resultSet = statement.executeQuery(sql);
while (resultSet.next()) {
String id = resultSet.getString(1);
providersIds.add(id);
} }
filters.add(filter);
return providersIds; SortedMap<Calendar, Info> timeSeries = getTimeSeries();
}finally { sortedMap.put(filter, timeSeries);
connection.close();
} filters.remove(filter);
}
return sortedMap;
*/
return null;
} }
@Override @Override
public List<UsageValue> getUsageValueQuotaTotal(List<UsageValue> listUsage) throws Exception { public List<UsageValue> getUsageValueQuotaTotal(List<UsageValue> listUsage) throws Exception {
return null; return null;
} }
@Override
public SortedMap<Filter, SortedMap<Calendar, Long>> getSpaceTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, List<String> providersId) throws Exception {
return null;
}
@Override @Override
public void close() throws Exception { public void close() throws Exception {

View File

@ -2,10 +2,12 @@ package org.gcube.accounting.analytics.persistence.postgresql;
import java.sql.SQLException; import java.sql.SQLException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Filter;
@ -14,6 +16,7 @@ import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.TemporalConstraint.CalendarEnum; import org.gcube.accounting.analytics.TemporalConstraint.CalendarEnum;
import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.UsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
import org.gcube.accounting.utility.postgresql.PostgreSQLQuery; import org.gcube.accounting.utility.postgresql.PostgreSQLQuery;
import org.gcube.accounting.utility.postgresql.RecordToDBFields; import org.gcube.accounting.utility.postgresql.RecordToDBFields;
@ -32,7 +35,7 @@ public class Query extends PostgreSQLQuery {
private Set<String> requestedTableField; private Set<String> requestedTableField;
protected TemporalConstraint temporalConstraint; protected TemporalConstraint temporalConstraint;
protected List<Filter> filters; protected Map<String, Collection<String>> filters;
protected Set<String> contexts; protected Set<String> contexts;
private String tableFieldToRequest; private String tableFieldToRequest;
@ -41,25 +44,8 @@ public class Query extends PostgreSQLQuery {
private String recordId; private String recordId;
protected String orConditionKey;
protected Set<String> orConditionValues;
public void setOrConditionKey(String orConditionKey) {
this.orConditionKey = orConditionKey;
}
public void setOrConditionValues(Collection<String> orConditionValues) {
this.orConditionValues = new HashSet<>(orConditionValues);
}
public void addOrConditionValue(String conditionValue) {
if(this.orConditionValues == null) {
this.orConditionValues = new HashSet<>();
}
this.orConditionValues.add(conditionValue);
}
public Query(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception { public Query(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
super();
this.clz = clz; this.clz = clz;
this.recordToDBFields = RecordToDBMapping.getRecordToDBFields(clz); this.recordToDBFields = RecordToDBMapping.getRecordToDBFields(clz);
} }
@ -72,8 +58,20 @@ public class Query extends PostgreSQLQuery {
this.temporalConstraint = temporalConstraint; this.temporalConstraint = temporalConstraint;
} }
public void setFilters(List<Filter> filters) { public void setFilters(Collection<? extends Filter> filters) {
this.filters = filters; this.filters = new HashMap<>();
if(filters!=null && filters.size()>0) {
for(Filter filter : filters) {
String key = filter.getKey();
String value = filter.getValue();
Collection<String> list = this.filters.get(key);
if(list == null) {
list = new ArrayList<>();
this.filters.put(key, list);
}
list.add(value);
}
}
} }
public void setContexts(Set<String> contexts) { public void setContexts(Set<String> contexts) {
@ -103,7 +101,7 @@ public class Query extends PostgreSQLQuery {
this.recordId = recordId; this.recordId = recordId;
} }
public RecordToDBFields getRecordToDBMapper() { public RecordToDBFields getRecordToDBFields() {
return recordToDBFields; return recordToDBFields;
} }
@ -123,26 +121,29 @@ public class Query extends PostgreSQLQuery {
protected void addFilters() throws SQLException { protected void addFilters() throws SQLException {
if(filters!=null && filters.size()>0) { if(filters!=null && filters.size()>0) {
// The first filter if the time_bucket if(filters.size()>1) {
stringBuffer.append(" AND "); stringBuffer.append("(");
}
boolean first = true; boolean first = true;
for(Filter filter : filters) { for(String key : filters.keySet()) {
if(first) { if(first) {
first = false; first = false;
}else { }else {
stringBuffer.append(" AND "); stringBuffer.append(" AND ");
} }
appendTableField(filter.getKey()); addOrConditions(key, filters.get(key));
stringBuffer.append("="); }
appendValue(filter.getValue()); if(filters.size()>1) {
stringBuffer.append(")");
} }
} }
} }
protected void addOrConditions(String key, Set<String> values) throws SQLException { protected void addOrConditions(String key, Collection<String> values) throws SQLException {
if(values!=null && values.size()>0) { if(values!=null && values.size()>0) {
// The first filter if the time_bucket if(values.size()>1) {
stringBuffer.append(" AND ("); stringBuffer.append("(");
}
boolean first = true; boolean first = true;
for(String value : values) { for(String value : values) {
if(first) { if(first) {
@ -154,19 +155,16 @@ public class Query extends PostgreSQLQuery {
stringBuffer.append("="); stringBuffer.append("=");
appendValue(value); appendValue(value);
} }
stringBuffer.append(")"); if(values.size()>1) {
stringBuffer.append(")");
}
} }
} }
protected void addContextFilter() throws SQLException { protected void addContextFilter() throws SQLException {
addOrConditions(UsageRecord.SCOPE, contexts); addOrConditions(UsageRecord.SCOPE, contexts);
} }
protected void addOrConditionFilter() throws SQLException {
addOrConditions(orConditionKey, orConditionValues);
}
protected void addEmittedFields(boolean addNested) throws Exception { protected void addEmittedFields(boolean addNested) throws Exception {
Set<String> aggregatedField = clz.newInstance().getAggregatedFields(); Set<String> aggregatedField = clz.newInstance().getAggregatedFields();
for(String fieldName : aggregatedField) { for(String fieldName : aggregatedField) {
@ -187,8 +185,22 @@ public class Query extends PostgreSQLQuery {
stringBuffer.append(") AS "); stringBuffer.append(") AS ");
break; break;
case AggregatedStorageStatusRecord.DATA_COUNT:
stringBuffer.append(", MAX(");
if(addNested) {
stringBuffer.append(NESTED_TABLE_NAME);
stringBuffer.append(".");
}
stringBuffer.append(dbField);
stringBuffer.append(") AS ");
break;
case AggregatedStorageUsageRecord.DATA_VOLUME: case AggregatedStorageUsageRecord.DATA_VOLUME:
stringBuffer.append(", SUM("); if(clz.isAssignableFrom(AggregatedStorageUsageRecord.class)) {
stringBuffer.append(", SUM(");
}else if(clz.isAssignableFrom(AggregatedStorageStatusRecord.class)){
stringBuffer.append(", MAX(");
}
if(addNested) { if(addNested) {
stringBuffer.append(NESTED_TABLE_NAME); stringBuffer.append(NESTED_TABLE_NAME);
stringBuffer.append("."); stringBuffer.append(".");
@ -262,7 +274,7 @@ public class Query extends PostgreSQLQuery {
} }
protected void addTemporalConstraintToQuery() throws SQLException { protected void addTemporalConstraintToQuery() throws SQLException {
stringBuffer.append(" WHERE "); stringBuffer.append(" WHERE (");
String tableField = getTableField(AggregatedRecord.START_TIME); String tableField = getTableField(AggregatedRecord.START_TIME);
stringBuffer.append(tableField); stringBuffer.append(tableField);
stringBuffer.append(" > "); stringBuffer.append(" > ");
@ -271,8 +283,9 @@ public class Query extends PostgreSQLQuery {
stringBuffer.append(" AND "); stringBuffer.append(" AND ");
tableField = getTableField(AggregatedRecord.END_TIME); tableField = getTableField(AggregatedRecord.END_TIME);
stringBuffer.append(tableField); stringBuffer.append(tableField);
stringBuffer.append(" < "); stringBuffer.append(" <= ");
appendValue(temporalConstraint.getAlignedEndTime()); appendValue(temporalConstraint.getAlignedEndTime());
stringBuffer.append(")");
} }
protected void addDateGroupBy(boolean addNested) { protected void addDateGroupBy(boolean addNested) {
@ -305,11 +318,9 @@ public class Query extends PostgreSQLQuery {
} }
protected void addLimit() { protected void addLimit() {
stringBuffer.append(" LIMIT ");
if(limit != null) { if(limit != null) {
stringBuffer.append(" LIMIT ");
stringBuffer.append(limit.toString()); stringBuffer.append(limit.toString());
}else {
stringBuffer.append(50);
} }
} }
@ -358,8 +369,15 @@ public class Query extends PostgreSQLQuery {
addTemporalConstraintToQuery(); addTemporalConstraintToQuery();
addFilters(); if(filters!=null && filters.size()>0) {
addContextFilter(); stringBuffer.append(" AND ");
addFilters();
}
if(contexts!=null && contexts.size()>0) {
stringBuffer.append(" AND ");
addContextFilter();
}
addDateGroupBy(false); addDateGroupBy(false);
addDateOrderBy(false); addDateOrderBy(false);
@ -383,10 +401,27 @@ public class Query extends PostgreSQLQuery {
stringBuffer.append(" FROM "); stringBuffer.append(" FROM ");
stringBuffer.append(recordToDBFields.getTableName()); stringBuffer.append(recordToDBFields.getTableName());
addTemporalConstraintToQuery(); boolean addAnd = false;
if(temporalConstraint!=null) {
addTemporalConstraintToQuery();
addAnd = true;
}
addFilters(); if(filters!=null && filters.size()>0) {
addContextFilter(); if(addAnd) {
stringBuffer.append(" AND ");
}
addFilters();
addAnd = true;
}
if(contexts!=null && contexts.size()>0) {
if(addAnd) {
stringBuffer.append(" AND ");
}
addContextFilter();
addAnd = false;
}
addGroupAndOrderByForOrderByField(); addGroupAndOrderByForOrderByField();
addLimit(); addLimit();

View File

@ -5,9 +5,12 @@ package org.gcube.accounting.analytics.persistence.postgresql;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.SortedMap; import java.util.SortedMap;
import java.util.SortedSet; import java.util.SortedSet;
@ -21,10 +24,13 @@ import org.gcube.accounting.analytics.exception.KeyException;
import org.gcube.accounting.analytics.exception.ValueException; import org.gcube.accounting.analytics.exception.ValueException;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.UsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
import org.gcube.accounting.datamodel.basetypes.AbstractStorageUsageRecord.DataType;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping; import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.Record;
@ -62,6 +68,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
protected TemporalConstraint getTemporalConstraint() { protected TemporalConstraint getTemporalConstraint() {
Calendar startTimeCalendar = Calendar.getInstance(); Calendar startTimeCalendar = Calendar.getInstance();
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
startTimeCalendar.set(Calendar.YEAR, 2015); startTimeCalendar.set(Calendar.YEAR, 2015);
startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY); startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY);
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
@ -69,6 +76,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
startTimeCalendar.set(Calendar.MINUTE, 0); startTimeCalendar.set(Calendar.MINUTE, 0);
Calendar entTimeCalendar = Calendar.getInstance(); Calendar entTimeCalendar = Calendar.getInstance();
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
entTimeCalendar.set(Calendar.YEAR, 2021); entTimeCalendar.set(Calendar.YEAR, 2021);
entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH); entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH);
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); entTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
@ -89,6 +97,231 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
return classes; return classes;
} }
@Test
public void testGetNoContextTimeSeriesYearly() throws Exception {
List<Filter> filters = getFilters();
AggregationMode aggregationMode = AggregationMode.YEARLY;
Calendar startTimeCalendar = Calendar.getInstance();
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
startTimeCalendar.set(Calendar.YEAR, 2015);
startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY);
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
startTimeCalendar.set(Calendar.MINUTE, 0);
Calendar entTimeCalendar = Calendar.getInstance();
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
entTimeCalendar.set(Calendar.YEAR, 2021);
entTimeCalendar.set(Calendar.MONTH, Calendar.DECEMBER);
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 31);
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
entTimeCalendar.set(Calendar.MINUTE, 0);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
temporalConstraint.setAggregationMode(aggregationMode);
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c);
logger.debug("{}", info);
}
}
@Test
public void testGetNoContextTimeSeriesMonthly() throws Exception {
List<Filter> filters = getFilters();
AggregationMode aggregationMode = AggregationMode.MONTHLY;
Calendar startTimeCalendar = Calendar.getInstance();
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
startTimeCalendar.set(Calendar.YEAR, 2015);
startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY);
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
startTimeCalendar.set(Calendar.MINUTE, 0);
Calendar entTimeCalendar = Calendar.getInstance();
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
entTimeCalendar.set(Calendar.YEAR, 2021);
entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
entTimeCalendar.set(Calendar.MINUTE, 0);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
temporalConstraint.setAggregationMode(aggregationMode);
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c);
logger.debug("{}", info);
}
}
@Test
public void testGetNoContextTimeSeriesDaily() throws Exception {
List<Filter> filters = getFilters();
AggregationMode aggregationMode = AggregationMode.DAILY;
Calendar startTimeCalendar = Calendar.getInstance();
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
startTimeCalendar.set(Calendar.YEAR, 2021);
startTimeCalendar.set(Calendar.MONTH, Calendar.OCTOBER);
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
startTimeCalendar.set(Calendar.MINUTE, 0);
Calendar entTimeCalendar = Calendar.getInstance();
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
entTimeCalendar.set(Calendar.YEAR, 2021);
entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
entTimeCalendar.set(Calendar.MINUTE, 0);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
temporalConstraint.setAggregationMode(aggregationMode);
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c);
logger.debug("{}", info);
}
}
@Test
public void testGetNoContextTimeSeriesHourly() throws Exception {
List<Filter> filters = getFilters();
AggregationMode aggregationMode = AggregationMode.HOURLY;
Calendar startTimeCalendar = Calendar.getInstance();
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
startTimeCalendar.set(Calendar.YEAR, 2021);
startTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 00);
startTimeCalendar.set(Calendar.MINUTE, 0);
Calendar entTimeCalendar = Calendar.getInstance();
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
entTimeCalendar.set(Calendar.YEAR, 2021);
entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 23);
entTimeCalendar.set(Calendar.MINUTE, 0);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
temporalConstraint.setAggregationMode(aggregationMode);
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c);
logger.debug("{}", info);
}
}
@Test
public void testGetNoContextTimeSeriesMinutely() throws Exception {
List<Filter> filters = getFilters();
AggregationMode aggregationMode = AggregationMode.MINUTELY;
Calendar startTimeCalendar = Calendar.getInstance();
startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
startTimeCalendar.set(Calendar.YEAR, 2021);
startTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 17);
startTimeCalendar.set(Calendar.MINUTE, 15);
Calendar entTimeCalendar = Calendar.getInstance();
entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
entTimeCalendar.set(Calendar.YEAR, 2021);
entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER);
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26);
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 17);
entTimeCalendar.set(Calendar.MINUTE, 15);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode);
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name());
temporalConstraint.setAggregationMode(aggregationMode);
accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c);
logger.debug("{}", info);
}
}
@Test @Test
public void testGetNoContextTimeSeries() throws Exception { public void testGetNoContextTimeSeries() throws Exception {
@ -102,7 +335,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
logger.debug("----------------------------------------------------------------------"); logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getNoContextTimeSeries(clz, temporalConstraint, filters); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) { for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c); Info info = timeseries.get(c);
logger.debug("{}", info); logger.debug("{}", info);
@ -123,7 +360,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(clz, temporalConstraint, filters); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) { for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c); Info info = timeseries.get(c);
logger.debug("{}", info); logger.debug("{}", info);
@ -142,15 +383,21 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
List<Class<? extends AggregatedRecord<?, ?>>> classes = getClassesToTest(); List<Class<? extends AggregatedRecord<?, ?>>> classes = getClassesToTest();
List<String> contexts = new ArrayList<>(); Set<String> contexts = new HashSet<>();
contexts.add("/gcube/devsec/devVRE"); contexts.add("/gcube/devsec/devVRE");
contexts.add("/gcube/devNext/NextNext"); contexts.add("/gcube/devNext/NextNext");
for(Class<? extends AggregatedRecord<?, ?>> clz : classes) { for(Class<? extends AggregatedRecord<?, ?>> clz : classes) {
logger.debug("----------------------------------------------------------------------"); logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
SortedMap<Filter, SortedMap<Calendar, Info>> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries(clz, temporalConstraint, filters, contexts); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setContexts(contexts);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Filter, SortedMap<Calendar, Info>> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries();
for(Filter f : contextTimeseries.keySet()) { for(Filter f : contextTimeseries.keySet()) {
logger.debug("{}", f); logger.debug("{}", f);
SortedMap<Calendar, Info> timeseries = contextTimeseries.get(f); SortedMap<Calendar, Info> timeseries = contextTimeseries.get(f);
@ -184,13 +431,31 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
String key = keysToRequest.get(clz); String key = keysToRequest.get(clz);
SortedSet<NumberedFilter> numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
SortedSet<NumberedFilter> numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key);
for(NumberedFilter numberedFilter : numberedFilters) { for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter); logger.debug("{}", numberedFilter);
} }
accountingPersistenceQueryPostgreSQL.setFilters(filters);
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key);
for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter);
}
accountingPersistenceQueryPostgreSQL.setFilters(null);
logger.debug("----Only first 3 results"); logger.debug("----Only first 3 results");
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key, 3); numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3);
for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter);
}
accountingPersistenceQueryPostgreSQL.setFilters(filters);
logger.debug("----Only first 3 results");
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3);
for(NumberedFilter numberedFilter : numberedFilters) { for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter); logger.debug("{}", numberedFilter);
} }
@ -207,7 +472,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
TemporalConstraint temporalConstraint = getTemporalConstraint(); TemporalConstraint temporalConstraint = getTemporalConstraint();
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty); accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedServiceUsageRecord.class);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty, null);
for(NumberedFilter numberedFilter : topTimeSeries.keySet()) { for(NumberedFilter numberedFilter : topTimeSeries.keySet()) {
logger.debug("{}", numberedFilter); logger.debug("{}", numberedFilter);
SortedMap<Calendar, Info> timeseries = topTimeSeries.get(numberedFilter); SortedMap<Calendar, Info> timeseries = topTimeSeries.get(numberedFilter);
@ -226,8 +495,63 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
} }
@Test @Test
public void testGetSpaceProvidersIds() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { public void storageStatusTest() throws Exception {
SortedSet<String> spaceProvidersIds = accountingPersistenceQueryPostgreSQL.getSpaceProvidersIds(); accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedStorageStatusRecord.class);
logger.debug("{}", spaceProvidersIds); Collection<NumberedFilter> myFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedStorageStatusRecord.DATA_TYPE);
accountingPersistenceQueryPostgreSQL.setFilters(myFilters);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(getTemporalConstraint());
accountingPersistenceQueryPostgreSQL.getTimeSeries();
} }
@Test
public void testGetSpaceTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
TemporalConstraint temporalConstraint = getTemporalConstraint();
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
Set<String> dataTypes = new HashSet<>();
dataTypes.add(DataType.STORAGE.name());
SortedMap<Filter, SortedMap<Calendar, Info>> getSpaceTimeSeries = accountingPersistenceQueryPostgreSQL.getSpaceTimeSeries(dataTypes);
for(Filter filter : getSpaceTimeSeries.keySet()) {
logger.debug("{}", filter);
SortedMap<Calendar, Info> timeseries = getSpaceTimeSeries.get(filter);
for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c);
logger.debug("{}", info);
}
}
}
@Test
public void testStorageTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
AccountingPersistenceQuery apq = AccountingPersistenceQueryFactory.getInstance();
Calendar startTimeCalendar = Calendar.getInstance();
startTimeCalendar.set(Calendar.YEAR, 2017);
startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY);
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
startTimeCalendar.set(Calendar.MINUTE, 0);
Calendar entTimeCalendar = Calendar.getInstance();
entTimeCalendar.set(Calendar.YEAR, 2021);
entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH);
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 22);
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
entTimeCalendar.set(Calendar.MINUTE, 0);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.DAILY);
apq.setRequestedRecords(AggregatedStorageStatusRecord.class);
apq.setTemporalConstraint(temporalConstraint);
SortedMap<Calendar, Info> timeseries = apq.getTimeSeries();
for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c);
logger.debug("{}", info);
}
}
} }

View File

@ -13,7 +13,6 @@ import java.util.Set;
import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.UsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
@ -60,6 +59,8 @@ public class QueryTest extends ContextTest {
List<Filter> filters = new ArrayList<>(); List<Filter> filters = new ArrayList<>();
Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser()); Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser());
filters.add(filter); filters.add(filter);
filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser());
filters.add(filter);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY);
Query query = new Query(AggregatedServiceUsageRecord.class); Query query = new Query(AggregatedServiceUsageRecord.class);
@ -69,9 +70,9 @@ public class QueryTest extends ContextTest {
logger.debug(ret); logger.debug(ret);
String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>(); Set<String> contexts = new HashSet<>();
contexts.add(currentScope); contexts.add(DEFAULT_TEST_SCOPE);
contexts.add(ALTERNATIVE_TEST_SCOPE);
query.setContexts(contexts); query.setContexts(contexts);
ret = query.getTimeSeriesQuery(); ret = query.getTimeSeriesQuery();
logger.debug(ret); logger.debug(ret);
@ -91,7 +92,7 @@ public class QueryTest extends ContextTest {
@Test @Test
public void testGetRecordQuery() throws Exception { public void testGetRecordQuery() throws Exception {
Query query = new Query(AggregatedServiceUsageRecord.class); Query query = new Query(AggregatedServiceUsageRecord.class);
query.setRecordId("fa573711-ceb6-44ba-9c83-bd47e0915b80"); query.setRecordId("7c3fcb31-6909-451d-bae0-633b3bec0c21");
String ret = query.getRecordQuery(); String ret = query.getRecordQuery();
logger.debug(ret); logger.debug(ret);
} }

View File

@ -10,7 +10,7 @@
<logger name="org.gcube" level="INFO" /> <logger name="org.gcube" level="INFO" />
<logger name="org.gcube.accounting.analytics.persistence.postgresql" level="TRACE" /> <logger name="org.gcube.accounting.analytics" level="TRACE" />
<root level="WARN"> <root level="WARN">
<appender-ref ref="STDOUT" /> <appender-ref ref="STDOUT" />