Merge branch 'feature/21353'

Conflicts:
	CHANGELOG.md
	pom.xml
	src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java
This commit is contained in:
Luca Frosini 2021-11-02 11:47:56 +01:00
commit 77b5290e53
6 changed files with 200 additions and 148 deletions

View File

@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for Accounting Analytics Backend Connector for PostgreSQL # Changelog for Accounting Analytics Backend Connector for PostgreSQL
## [v2.0.0-SNAPSHOT]
- Added possibility of having multiple values for the same key which will be used in OR [#21353]
## [v1.0.1] ## [v1.0.1]
- Fixed bug on query - Fixed bug on query

View File

@ -9,7 +9,7 @@
</parent> </parent>
<groupId>org.gcube.accounting</groupId> <groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics-persistence-postgresql</artifactId> <artifactId>accounting-analytics-persistence-postgresql</artifactId>
<version>1.0.1</version> <version>2.0.0-SNAPSHOT</version>
<name>Accounting Analytics Backend Connector for PostgreSQL</name> <name>Accounting Analytics Backend Connector for PostgreSQL</name>
<description>Accounting Analytics Backend Connector for PostgreSQL</description> <description>Accounting Analytics Backend Connector for PostgreSQL</description>
@ -44,7 +44,7 @@
<dependency> <dependency>
<groupId>org.gcube.accounting</groupId> <groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics</artifactId> <artifactId>accounting-analytics</artifactId>
<version>[3.0.0, 4.0.0-SNAPSHOT)</version> <version>[4.0.0-SNAPSHOT, 5.0.0)</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.gcube.accounting</groupId> <groupId>org.gcube.accounting</groupId>

View File

@ -23,23 +23,20 @@ import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.NumberedFilter;
import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.UsageValue; import org.gcube.accounting.analytics.UsageValue;
import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException;
import org.gcube.accounting.analytics.exception.KeyException;
import org.gcube.accounting.analytics.exception.ValueException;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; import org.gcube.accounting.persistence.AccountingPersistenceConfiguration;
import org.gcube.accounting.utility.postgresql.RecordToDBConnection; import org.gcube.accounting.utility.postgresql.RecordToDBConnection;
import org.gcube.accounting.utility.postgresql.RecordToDBFields; import org.gcube.accounting.utility.postgresql.RecordToDBFields;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping; import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.records.RecordUtility; import org.gcube.documentstore.records.RecordUtility;
import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -58,6 +55,39 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
protected AccountingPersistenceBackendQueryConfiguration configuration; protected AccountingPersistenceBackendQueryConfiguration configuration;
protected ObjectMapper objectMapper;
protected Class<? extends AggregatedRecord<?, ?>> clz;
protected TemporalConstraint temporalConstraint;
protected Set<String> contexts;
protected Set<Filter> filters;
public AccountingPersistenceQueryPostgreSQL() {
objectMapper = new ObjectMapper();
}
@Override
public void setRequestedRecords(Class<? extends AggregatedRecord<?, ?>> clz) {
this.clz = clz;
}
@Override
public void setTemporalConstraint(TemporalConstraint temporalConstraint) {
this.temporalConstraint = temporalConstraint;
}
@Override
public void setContexts(Set<String> contexts) {
this.contexts = contexts;
}
@Override
public void setFilters(Set<Filter> filters) {
this.filters = filters;
}
static { static {
// One Record per package is enough // One Record per package is enough
RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage()); RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage());
@ -87,9 +117,24 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
} }
} }
protected SortedMap<Calendar, Info> getTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz, protected void addProperty(ObjectNode objectNode, String key, Object value) {
TemporalConstraint temporalConstraint, List<Filter> filters, Set<String> contexts) if(value instanceof Number) {
throws Exception {
if(value instanceof Integer) {
objectNode.put(key, (int) value);
return;
}
Long longValue = Long.valueOf(value.toString());
objectNode.put(key, longValue);
return;
}
objectNode.put(key, (String) value.toString());
}
protected SortedMap<Calendar, Info> getTimeSeries(Set<String> contexts) throws Exception {
Connection connection = getConnection(clz); Connection connection = getConnection(clz);
try { try {
Statement statement = connection.createStatement(); Statement statement = connection.createStatement();
@ -114,15 +159,15 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
OffsetDateTime offsetDateTime = resultSet.getObject(tableFieldName, OffsetDateTime.class); OffsetDateTime offsetDateTime = resultSet.getObject(tableFieldName, OffsetDateTime.class);
Calendar calendar = getCalendar(offsetDateTime); Calendar calendar = getCalendar(offsetDateTime);
JSONObject jsonObject = new JSONObject(); ObjectNode objectNode = objectMapper.createObjectNode();
for(String tableField : requestedTableField) { for(String tableField : requestedTableField) {
String usageRecordField = recordToDBMapper.getRecordField(tableField); String usageRecordField = recordToDBMapper.getRecordField(tableField);
Object object = resultSet.getObject(tableField); Object obj = resultSet.getObject(tableField);
jsonObject.put(usageRecordField, object); addProperty(objectNode, usageRecordField, obj);
} }
Info info = new Info(calendar, jsonObject); Info info = new Info(calendar, objectNode);
result.put(calendar, info); result.put(calendar, info);
} }
@ -133,13 +178,8 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
} }
@Override @Override
public SortedMap<Calendar, Info> getTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz, public SortedMap<Calendar, Info> getTimeSeries() throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters) return getTimeSeries(contexts);
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>();
contexts.add(context);
return getTimeSeries(clz, temporalConstraint, filters, contexts);
} }
protected Calendar getCalendar(OffsetDateTime offsetDateTime) { protected Calendar getCalendar(OffsetDateTime offsetDateTime) {
@ -148,25 +188,17 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
calendar.setTimeInMillis(epochMillis); calendar.setTimeInMillis(epochMillis);
return calendar; return calendar;
} }
@Override
public SortedMap<Calendar, Info> getNoContextTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters)
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
return getTimeSeries(clz, temporalConstraint, filters, null);
}
@Override @Override
public SortedMap<Filter, SortedMap<Calendar, Info>> getContextTimeSeries( public SortedMap<Filter, SortedMap<Calendar, Info>> getContextTimeSeries() throws Exception {
Class<? extends AggregatedRecord<?, ?>> clz, TemporalConstraint temporalConstraint, List<Filter> filters,
List<String> contexts) throws Exception {
SortedMap<Filter,SortedMap<Calendar,Info>> ret = new TreeMap<>(); SortedMap<Filter,SortedMap<Calendar,Info>> ret = new TreeMap<>();
for(String context : contexts) { for(String context : contexts) {
Filter contextFilter = new Filter("scope", context); Filter contextFilter = new Filter("scope", context);
Set<String> timeSeriesContexts = new HashSet<>(); Set<String> ctxs = new HashSet<>();
timeSeriesContexts.add(context); ctxs.add(context);
SortedMap<Calendar, Info> timeSeries = getTimeSeries(clz, temporalConstraint, filters, timeSeriesContexts);
SortedMap<Calendar, Info> timeSeries = getTimeSeries(ctxs);
if(!timeSeries.isEmpty()) { if(!timeSeries.isEmpty()) {
ret.put(contextFilter, timeSeries); ret.put(contextFilter, timeSeries);
} }
@ -174,9 +206,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
return ret; return ret;
} }
protected SortedSet<NumberedFilter> getNumberedValues(Class<? extends AggregatedRecord<?, ?>> clz, protected SortedSet<NumberedFilter> getNumberedValues(String key, String orderingProperty, Integer limit) throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters, String key,
String orderingProperty, Integer limit) throws Exception {
Connection connection = getConnection(clz); Connection connection = getConnection(clz);
try { try {
Statement statement = connection.createStatement(); Statement statement = connection.createStatement();
@ -191,10 +221,6 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
SortedSet<NumberedFilter> result = new TreeSet<>(); SortedSet<NumberedFilter> result = new TreeSet<>();
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>();
contexts.add(context);
Query query = new Query(clz); Query query = new Query(clz);
query.setTemporalConstraint(temporalConstraint); query.setTemporalConstraint(temporalConstraint);
query.setFilters(filters); query.setFilters(filters);
@ -229,34 +255,26 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
} }
@Override @Override
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz, public SortedSet<NumberedFilter> getFilterValues(String key) throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters, String key) throws Exception { return getNumberedValues(key, null, null);
return getNumberedValues(clz, temporalConstraint, filters, key, null, null);
} }
@Override @Override
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz, public SortedSet<NumberedFilter> getFilterValues(String key, Integer limit) throws Exception {
TemporalConstraint temporalConstraint, List<Filter> filters, String key, Integer limit) throws Exception { return getNumberedValues(key, null, limit);
return getNumberedValues(clz, temporalConstraint, filters, key, null, limit);
} }
@Override @Override
public SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues( public SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues(String topKey, String orderingProperty)
Class<? extends AggregatedRecord<?, ?>> clz, TemporalConstraint temporalConstraint, List<Filter> filters, throws Exception {
String topKey, String orderingProperty)
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
String context = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>();
contexts.add(context);
SortedMap<NumberedFilter,SortedMap<Calendar,Info>> ret = new TreeMap<>(); SortedMap<NumberedFilter,SortedMap<Calendar,Info>> ret = new TreeMap<>();
SortedSet<NumberedFilter> top = getNumberedValues(clz, temporalConstraint, filters, topKey, orderingProperty, 10); SortedSet<NumberedFilter> top = getNumberedValues(topKey, orderingProperty, 10);
for(NumberedFilter numberedFilter : top) { for(NumberedFilter numberedFilter : top) {
filters.add(numberedFilter); filters.add(numberedFilter);
SortedMap<Calendar,Info> map = getTimeSeries(clz, temporalConstraint, filters, contexts); SortedMap<Calendar,Info> map = getTimeSeries();
ret.put(numberedFilter, map); ret.put(numberedFilter, map);
filters.remove(numberedFilter); filters.remove(numberedFilter);
} }
@ -309,42 +327,32 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
} }
@Override @Override
public SortedSet<String> getSpaceProvidersIds() throws Exception { public SortedMap<Filter, SortedMap<Calendar, Info>> getSpaceTimeSeries(Set<String> dataTypes) throws Exception {
Class<? extends AggregatedRecord<?, ?>> aggregatedRecordClass = AggregatedStorageStatusRecord.class; /*
Connection connection = getConnection(aggregatedRecordClass); SortedMap<Filter, SortedMap<Calendar, Info>> sortedMap = new TreeMap<>();
try { setRequestedRecords(AggregatedStorageStatusRecord.class);
Statement statement = connection.createStatement(); for(String dataType : dataTypes) {
Filter filter = new Filter(StorageStatusRecord.DATA_TYPE, dataType);
Query query = new Query(aggregatedRecordClass); if(filters == null) {
query.setTableFieldToRequest(AggregatedStorageStatusRecord.PROVIDER_URI); filters = new HashSet<>();
String sql = query.getDinstinctValuesQuery();
SortedSet<String> providersIds = new TreeSet<>();
logger.trace("Going to request the following query: {}", sql);
ResultSet resultSet = statement.executeQuery(sql);
while (resultSet.next()) {
String id = resultSet.getString(1);
providersIds.add(id);
} }
filters.add(filter);
return providersIds; SortedMap<Calendar, Info> timeSeries = getTimeSeries();
}finally { sortedMap.put(filter, timeSeries);
connection.close();
} filters.remove(filter);
}
return sortedMap;
*/
return null;
} }
@Override @Override
public List<UsageValue> getUsageValueQuotaTotal(List<UsageValue> listUsage) throws Exception { public List<UsageValue> getUsageValueQuotaTotal(List<UsageValue> listUsage) throws Exception {
return null; return null;
} }
@Override
public SortedMap<Filter, SortedMap<Calendar, Long>> getSpaceTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, List<String> providersId) throws Exception {
return null;
}
@Override @Override
public void close() throws Exception { public void close() throws Exception {

View File

@ -2,10 +2,12 @@ package org.gcube.accounting.analytics.persistence.postgresql;
import java.sql.SQLException; import java.sql.SQLException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Filter;
@ -32,7 +34,7 @@ public class Query extends PostgreSQLQuery {
private Set<String> requestedTableField; private Set<String> requestedTableField;
protected TemporalConstraint temporalConstraint; protected TemporalConstraint temporalConstraint;
protected List<Filter> filters; protected Map<String, Collection<String>> filters;
protected Set<String> contexts; protected Set<String> contexts;
private String tableFieldToRequest; private String tableFieldToRequest;
@ -41,24 +43,6 @@ public class Query extends PostgreSQLQuery {
private String recordId; private String recordId;
protected String orConditionKey;
protected Set<String> orConditionValues;
public void setOrConditionKey(String orConditionKey) {
this.orConditionKey = orConditionKey;
}
public void setOrConditionValues(Collection<String> orConditionValues) {
this.orConditionValues = new HashSet<>(orConditionValues);
}
public void addOrConditionValue(String conditionValue) {
if(this.orConditionValues == null) {
this.orConditionValues = new HashSet<>();
}
this.orConditionValues.add(conditionValue);
}
public Query(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception { public Query(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
this.clz = clz; this.clz = clz;
this.recordToDBFields = RecordToDBMapping.getRecordToDBFields(clz); this.recordToDBFields = RecordToDBMapping.getRecordToDBFields(clz);
@ -72,8 +56,20 @@ public class Query extends PostgreSQLQuery {
this.temporalConstraint = temporalConstraint; this.temporalConstraint = temporalConstraint;
} }
public void setFilters(List<Filter> filters) { public void setFilters(Collection<Filter> filters) {
this.filters = filters; this.filters = new HashMap<>();
if(filters!=null && filters.size()>0) {
for(Filter filter : filters) {
String key = filter.getKey();
String value = filter.getValue();
Collection<String> list = this.filters.get(key);
if(list == null) {
list = new ArrayList<>();
this.filters.put(key, list);
}
list.add(value);
}
}
} }
public void setContexts(Set<String> contexts) { public void setContexts(Set<String> contexts) {
@ -123,26 +119,29 @@ public class Query extends PostgreSQLQuery {
protected void addFilters() throws SQLException { protected void addFilters() throws SQLException {
if(filters!=null && filters.size()>0) { if(filters!=null && filters.size()>0) {
// The first filter if the time_bucket if(filters.size()>1) {
stringBuffer.append(" AND "); stringBuffer.append("(");
}
boolean first = true; boolean first = true;
for(Filter filter : filters) { for(String key : filters.keySet()) {
if(first) { if(first) {
first = false; first = false;
}else { }else {
stringBuffer.append(" AND "); stringBuffer.append(" AND ");
} }
appendTableField(filter.getKey()); addOrConditions(key, filters.get(key));
stringBuffer.append("="); }
appendValue(filter.getValue()); if(filters.size()>1) {
stringBuffer.append(")");
} }
} }
} }
protected void addOrConditions(String key, Set<String> values) throws SQLException { protected void addOrConditions(String key, Collection<String> values) throws SQLException {
if(values!=null && values.size()>0) { if(values!=null && values.size()>0) {
// The first filter if the time_bucket if(values.size()>1) {
stringBuffer.append(" AND ("); stringBuffer.append("(");
}
boolean first = true; boolean first = true;
for(String value : values) { for(String value : values) {
if(first) { if(first) {
@ -154,19 +153,16 @@ public class Query extends PostgreSQLQuery {
stringBuffer.append("="); stringBuffer.append("=");
appendValue(value); appendValue(value);
} }
stringBuffer.append(")"); if(values.size()>1) {
stringBuffer.append(")");
}
} }
} }
protected void addContextFilter() throws SQLException { protected void addContextFilter() throws SQLException {
addOrConditions(UsageRecord.SCOPE, contexts); addOrConditions(UsageRecord.SCOPE, contexts);
} }
protected void addOrConditionFilter() throws SQLException {
addOrConditions(orConditionKey, orConditionValues);
}
protected void addEmittedFields(boolean addNested) throws Exception { protected void addEmittedFields(boolean addNested) throws Exception {
Set<String> aggregatedField = clz.newInstance().getAggregatedFields(); Set<String> aggregatedField = clz.newInstance().getAggregatedFields();
for(String fieldName : aggregatedField) { for(String fieldName : aggregatedField) {
@ -262,7 +258,7 @@ public class Query extends PostgreSQLQuery {
} }
protected void addTemporalConstraintToQuery() throws SQLException { protected void addTemporalConstraintToQuery() throws SQLException {
stringBuffer.append(" WHERE "); stringBuffer.append(" WHERE (");
String tableField = getTableField(AggregatedRecord.START_TIME); String tableField = getTableField(AggregatedRecord.START_TIME);
stringBuffer.append(tableField); stringBuffer.append(tableField);
stringBuffer.append(" > "); stringBuffer.append(" > ");
@ -273,6 +269,7 @@ public class Query extends PostgreSQLQuery {
stringBuffer.append(tableField); stringBuffer.append(tableField);
stringBuffer.append(" < "); stringBuffer.append(" < ");
appendValue(temporalConstraint.getAlignedEndTime()); appendValue(temporalConstraint.getAlignedEndTime());
stringBuffer.append(")");
} }
protected void addDateGroupBy(boolean addNested) { protected void addDateGroupBy(boolean addNested) {
@ -358,7 +355,9 @@ public class Query extends PostgreSQLQuery {
addTemporalConstraintToQuery(); addTemporalConstraintToQuery();
stringBuffer.append(" AND ");
addFilters(); addFilters();
stringBuffer.append(" AND ");
addContextFilter(); addContextFilter();
addDateGroupBy(false); addDateGroupBy(false);
@ -385,7 +384,9 @@ public class Query extends PostgreSQLQuery {
addTemporalConstraintToQuery(); addTemporalConstraintToQuery();
stringBuffer.append(" AND ");
addFilters(); addFilters();
stringBuffer.append(" AND ");
addContextFilter(); addContextFilter();
addGroupAndOrderByForOrderByField(); addGroupAndOrderByForOrderByField();

View File

@ -6,8 +6,10 @@ package org.gcube.accounting.analytics.persistence.postgresql;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.SortedMap; import java.util.SortedMap;
import java.util.SortedSet; import java.util.SortedSet;
@ -25,6 +27,7 @@ import org.gcube.accounting.datamodel.UsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
import org.gcube.accounting.datamodel.basetypes.AbstractStorageUsageRecord.DataType;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping; import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.Record;
@ -51,8 +54,8 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
accountingPersistenceQueryPostgreSQL.prepareConnection(configuration); accountingPersistenceQueryPostgreSQL.prepareConnection(configuration);
} }
protected List<Filter> getFilters(){ protected Set<Filter> getFilters(){
List<Filter> filters = new ArrayList<>(); Set<Filter> filters = new HashSet<>();
String user = QueryTest.getRandomUser(); String user = QueryTest.getRandomUser();
logger.debug("Going to query filtering with user {}", user); logger.debug("Going to query filtering with user {}", user);
Filter filter = new Filter(UsageRecord.CONSUMER_ID, user); Filter filter = new Filter(UsageRecord.CONSUMER_ID, user);
@ -92,7 +95,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
@Test @Test
public void testGetNoContextTimeSeries() throws Exception { public void testGetNoContextTimeSeries() throws Exception {
List<Filter> filters = getFilters(); Set<Filter> filters = getFilters();
TemporalConstraint temporalConstraint = getTemporalConstraint(); TemporalConstraint temporalConstraint = getTemporalConstraint();
@ -102,7 +105,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
logger.debug("----------------------------------------------------------------------"); logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getNoContextTimeSeries(clz, temporalConstraint, filters); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) { for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c); Info info = timeseries.get(c);
logger.debug("{}", info); logger.debug("{}", info);
@ -112,7 +119,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
@Test @Test
public void testTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { public void testTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
List<Filter> filters = getFilters(); Set<Filter> filters = getFilters();
TemporalConstraint temporalConstraint = getTemporalConstraint(); TemporalConstraint temporalConstraint = getTemporalConstraint();
@ -123,7 +130,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(clz, temporalConstraint, filters); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Calendar, Info> timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries();
for(Calendar c : timeseries.keySet()) { for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c); Info info = timeseries.get(c);
logger.debug("{}", info); logger.debug("{}", info);
@ -136,21 +147,27 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
@Test @Test
public void testContextTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { public void testContextTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
List<Filter> filters = getFilters(); Set<Filter> filters = getFilters();
TemporalConstraint temporalConstraint = getTemporalConstraint(); TemporalConstraint temporalConstraint = getTemporalConstraint();
List<Class<? extends AggregatedRecord<?, ?>>> classes = getClassesToTest(); List<Class<? extends AggregatedRecord<?, ?>>> classes = getClassesToTest();
List<String> contexts = new ArrayList<>(); Set<String> contexts = new HashSet<>();
contexts.add("/gcube/devsec/devVRE"); contexts.add("/gcube/devsec/devVRE");
contexts.add("/gcube/devNext/NextNext"); contexts.add("/gcube/devNext/NextNext");
for(Class<? extends AggregatedRecord<?, ?>> clz : classes) { for(Class<? extends AggregatedRecord<?, ?>> clz : classes) {
logger.debug("----------------------------------------------------------------------"); logger.debug("----------------------------------------------------------------------");
logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz));
SortedMap<Filter, SortedMap<Calendar, Info>> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries(clz, temporalConstraint, filters, contexts); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setContexts(contexts);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<Filter, SortedMap<Calendar, Info>> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries();
for(Filter f : contextTimeseries.keySet()) { for(Filter f : contextTimeseries.keySet()) {
logger.debug("{}", f); logger.debug("{}", f);
SortedMap<Calendar, Info> timeseries = contextTimeseries.get(f); SortedMap<Calendar, Info> timeseries = contextTimeseries.get(f);
@ -167,7 +184,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
@Test @Test
public void testGetFilterValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { public void testGetFilterValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
List<Filter> filters = getFilters(); Set<Filter> filters = getFilters();
TemporalConstraint temporalConstraint = getTemporalConstraint(); TemporalConstraint temporalConstraint = getTemporalConstraint();
@ -184,13 +201,17 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
String key = keysToRequest.get(clz); String key = keysToRequest.get(clz);
SortedSet<NumberedFilter> numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedSet<NumberedFilter> numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key);
for(NumberedFilter numberedFilter : numberedFilters) { for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter); logger.debug("{}", numberedFilter);
} }
logger.debug("----Only first 3 results"); logger.debug("----Only first 3 results");
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key, 3); numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3);
for(NumberedFilter numberedFilter : numberedFilters) { for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter); logger.debug("{}", numberedFilter);
} }
@ -201,13 +222,17 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
public void testTopValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { public void testTopValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
String orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(AggregatedServiceUsageRecord.class); String orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(AggregatedServiceUsageRecord.class);
List<Filter> filters = new ArrayList<>(); Set<Filter> filters = new HashSet<>();
Filter filter = new Filter(UsageRecord.CONSUMER_ID, QueryTest.getRandomUser()); Filter filter = new Filter(UsageRecord.CONSUMER_ID, QueryTest.getRandomUser());
filters.add(filter); filters.add(filter);
TemporalConstraint temporalConstraint = getTemporalConstraint(); TemporalConstraint temporalConstraint = getTemporalConstraint();
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty); accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedServiceUsageRecord.class);
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
accountingPersistenceQueryPostgreSQL.setFilters(filters);
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty);
for(NumberedFilter numberedFilter : topTimeSeries.keySet()) { for(NumberedFilter numberedFilter : topTimeSeries.keySet()) {
logger.debug("{}", numberedFilter); logger.debug("{}", numberedFilter);
SortedMap<Calendar, Info> timeseries = topTimeSeries.get(numberedFilter); SortedMap<Calendar, Info> timeseries = topTimeSeries.get(numberedFilter);
@ -226,8 +251,23 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
} }
@Test @Test
public void testGetSpaceProvidersIds() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { public void testGetSpaceTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
SortedSet<String> spaceProvidersIds = accountingPersistenceQueryPostgreSQL.getSpaceProvidersIds();
logger.debug("{}", spaceProvidersIds); TemporalConstraint temporalConstraint = getTemporalConstraint();
accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint);
Set<String> dataTypes = new HashSet<>();
dataTypes.add(DataType.STORAGE.name());
SortedMap<Filter, SortedMap<Calendar, Info>> getSpaceTimeSeries = accountingPersistenceQueryPostgreSQL.getSpaceTimeSeries(dataTypes);
for(Filter filter : getSpaceTimeSeries.keySet()) {
logger.debug("{}", filter);
SortedMap<Calendar, Info> timeseries = getSpaceTimeSeries.get(filter);
for(Calendar c : timeseries.keySet()) {
Info info = timeseries.get(c);
logger.debug("{}", info);
}
}
} }
} }

View File

@ -3,17 +3,14 @@
*/ */
package org.gcube.accounting.analytics.persistence.postgresql; package org.gcube.accounting.analytics.persistence.postgresql;
import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Random; import java.util.Random;
import java.util.Set; import java.util.Set;
import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.UsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
@ -57,9 +54,11 @@ public class QueryTest extends ContextTest {
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 16); entTimeCalendar.set(Calendar.HOUR_OF_DAY, 16);
entTimeCalendar.set(Calendar.MINUTE, 17); entTimeCalendar.set(Calendar.MINUTE, 17);
List<Filter> filters = new ArrayList<>(); Set<Filter> filters = new HashSet<>();
Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser()); Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser());
filters.add(filter); filters.add(filter);
filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser());
filters.add(filter);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY);
Query query = new Query(AggregatedServiceUsageRecord.class); Query query = new Query(AggregatedServiceUsageRecord.class);
@ -69,9 +68,9 @@ public class QueryTest extends ContextTest {
logger.debug(ret); logger.debug(ret);
String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>(); Set<String> contexts = new HashSet<>();
contexts.add(currentScope); contexts.add(DEFAULT_TEST_SCOPE);
contexts.add(ALTERNATIVE_TEST_SCOPE);
query.setContexts(contexts); query.setContexts(contexts);
ret = query.getTimeSeriesQuery(); ret = query.getTimeSeriesQuery();
logger.debug(ret); logger.debug(ret);
@ -91,7 +90,7 @@ public class QueryTest extends ContextTest {
@Test @Test
public void testGetRecordQuery() throws Exception { public void testGetRecordQuery() throws Exception {
Query query = new Query(AggregatedServiceUsageRecord.class); Query query = new Query(AggregatedServiceUsageRecord.class);
query.setRecordId("fa573711-ceb6-44ba-9c83-bd47e0915b80"); query.setRecordId("7c3fcb31-6909-451d-bae0-633b3bec0c21");
String ret = query.getRecordQuery(); String ret = query.getRecordQuery();
logger.debug(ret); logger.debug(ret);
} }