From 88737307f0b9dca9a720e38f69ff1c4bddd841c1 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Fri, 7 May 2021 14:06:32 +0200 Subject: [PATCH 1/5] Switching to new accounting-analytics APIs --- CHANGELOG.md | 3 + pom.xml | 4 +- .../AccountingPersistenceQueryPostgreSQL.java | 174 +++++++++--------- .../persistence/postgresql/Query.java | 5 +- ...ountingPersistenceQueryPostgreSQLTest.java | 76 ++++++-- .../persistence/postgresql/QueryTest.java | 4 +- 6 files changed, 157 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c82d990..b3ff17f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for Accounting Analytics Backend Connector for PostgreSQL +## [v1.1.0-SNAPSHOT] + +- ## [v1.0.0] diff --git a/pom.xml b/pom.xml index 83659d4..387eb4c 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.accounting accounting-analytics-persistence-postgresql - 1.0.0 + 2.0.0-SNAPSHOT Accounting Analytics Backend Connector for PostgreSQL Accounting Analytics Backend Connector for PostgreSQL @@ -44,7 +44,7 @@ org.gcube.accounting accounting-analytics - [3.0.0, 4.0.0-SNAPSHOT) + [4.0.0-SNAPSHOT, 5.0.0) org.gcube.accounting diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java index 3f87586..0acaa51 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java @@ -23,23 +23,20 @@ import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.UsageValue; -import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException; -import org.gcube.accounting.analytics.exception.KeyException; -import org.gcube.accounting.analytics.exception.ValueException; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; -import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; import org.gcube.accounting.utility.postgresql.RecordToDBConnection; import org.gcube.accounting.utility.postgresql.RecordToDBFields; import org.gcube.accounting.utility.postgresql.RecordToDBMapping; +import org.gcube.com.fasterxml.jackson.databind.ObjectMapper; +import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.RecordUtility; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +55,39 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten protected AccountingPersistenceBackendQueryConfiguration configuration; + protected ObjectMapper objectMapper; + + protected Class> clz; + protected TemporalConstraint temporalConstraint; + + protected Set contexts; + protected Set filters; + + + public AccountingPersistenceQueryPostgreSQL() { + objectMapper = new ObjectMapper(); + } + + @Override + public void setRequestedRecords(Class> clz) { + this.clz = clz; + } + + @Override + public void setTemporalConstraint(TemporalConstraint temporalConstraint) { + this.temporalConstraint = temporalConstraint; + } + + @Override + public void setContexts(Set contexts) { + this.contexts = contexts; + } + + @Override + public void setFilters(Set filters) { + this.filters = filters; + } + static { // One Record per package is enough RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage()); @@ -87,9 +117,24 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } } - protected SortedMap getTimeSeries(Class> clz, - TemporalConstraint temporalConstraint, List filters, Set contexts) - throws Exception { + protected void addProperty(ObjectNode objectNode, String key, Object value) { + if(value instanceof Number) { + + if(value instanceof Integer) { + objectNode.put(key, (int) value); + return; + } + + Long longValue = Long.valueOf(value.toString()); + objectNode.put(key, longValue); + return; + } + + objectNode.put(key, (String) value.toString()); + + } + + protected SortedMap getTimeSeries(Set contexts) throws Exception { Connection connection = getConnection(clz); try { Statement statement = connection.createStatement(); @@ -114,15 +159,15 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten OffsetDateTime offsetDateTime = resultSet.getObject(tableFieldName, OffsetDateTime.class); Calendar calendar = getCalendar(offsetDateTime); - JSONObject jsonObject = new JSONObject(); + ObjectNode objectNode = objectMapper.createObjectNode(); for(String tableField : requestedTableField) { String usageRecordField = recordToDBMapper.getRecordField(tableField); - Object object = resultSet.getObject(tableField); - jsonObject.put(usageRecordField, object); + Object obj = resultSet.getObject(tableField); + addProperty(objectNode, usageRecordField, obj); } - Info info = new Info(calendar, jsonObject); + Info info = new Info(calendar, objectNode); result.put(calendar, info); } @@ -133,13 +178,8 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } @Override - public SortedMap getTimeSeries(Class> clz, - TemporalConstraint temporalConstraint, List filters) - throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - String context = AccountingPersistenceBackendQuery.getScopeToQuery(); - Set contexts = new HashSet<>(); - contexts.add(context); - return getTimeSeries(clz, temporalConstraint, filters, contexts); + public SortedMap getTimeSeries() throws Exception { + return getTimeSeries(contexts); } protected Calendar getCalendar(OffsetDateTime offsetDateTime) { @@ -148,25 +188,17 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten calendar.setTimeInMillis(epochMillis); return calendar; } - - @Override - public SortedMap getNoContextTimeSeries(Class> clz, - TemporalConstraint temporalConstraint, List filters) - throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - return getTimeSeries(clz, temporalConstraint, filters, null); - } @Override - public SortedMap> getContextTimeSeries( - Class> clz, TemporalConstraint temporalConstraint, List filters, - List contexts) throws Exception { + public SortedMap> getContextTimeSeries() throws Exception { SortedMap> ret = new TreeMap<>(); for(String context : contexts) { Filter contextFilter = new Filter("context", context); - Set timeSeriesContexts = new HashSet<>(); - timeSeriesContexts.add(context); - SortedMap timeSeries = getTimeSeries(clz, temporalConstraint, filters, timeSeriesContexts); + Set ctxs = new HashSet<>(); + ctxs.add(context); + + SortedMap timeSeries = getTimeSeries(ctxs); if(!timeSeries.isEmpty()) { ret.put(contextFilter, timeSeries); } @@ -174,9 +206,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten return ret; } - protected SortedSet getNumberedValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key, - String orderingProperty, Integer limit) throws Exception { + protected SortedSet getNumberedValues(String key, String orderingProperty, Integer limit) throws Exception { Connection connection = getConnection(clz); try { Statement statement = connection.createStatement(); @@ -191,10 +221,6 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten SortedSet result = new TreeSet<>(); - String context = AccountingPersistenceBackendQuery.getScopeToQuery(); - Set contexts = new HashSet<>(); - contexts.add(context); - Query query = new Query(clz); query.setTemporalConstraint(temporalConstraint); query.setFilters(filters); @@ -229,34 +255,26 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } @Override - public SortedSet getFilterValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key) throws Exception { - return getNumberedValues(clz, temporalConstraint, filters, key, null, null); + public SortedSet getFilterValues(String key) throws Exception { + return getNumberedValues(key, null, null); } @Override - public SortedSet getFilterValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key, Integer limit) throws Exception { - return getNumberedValues(clz, temporalConstraint, filters, key, null, limit); + public SortedSet getFilterValues(String key, Integer limit) throws Exception { + return getNumberedValues(key, null, limit); } @Override - public SortedMap> getTopValues( - Class> clz, TemporalConstraint temporalConstraint, List filters, - String topKey, String orderingProperty) - throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - - String context = AccountingPersistenceBackendQuery.getScopeToQuery(); - Set contexts = new HashSet<>(); - contexts.add(context); + public SortedMap> getTopValues(String topKey, String orderingProperty) + throws Exception { SortedMap> ret = new TreeMap<>(); - SortedSet top = getNumberedValues(clz, temporalConstraint, filters, topKey, orderingProperty, 10); + SortedSet top = getNumberedValues(topKey, orderingProperty, 10); for(NumberedFilter numberedFilter : top) { filters.add(numberedFilter); - SortedMap map = getTimeSeries(clz, temporalConstraint, filters, contexts); + SortedMap map = getTimeSeries(); ret.put(numberedFilter, map); filters.remove(numberedFilter); } @@ -309,42 +327,32 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } @Override - public SortedSet getSpaceProvidersIds() throws Exception { - Class> aggregatedRecordClass = AggregatedStorageStatusRecord.class; - Connection connection = getConnection(aggregatedRecordClass); - try { - Statement statement = connection.createStatement(); - - Query query = new Query(aggregatedRecordClass); - query.setTableFieldToRequest(AggregatedStorageStatusRecord.PROVIDER_ID); - String sql = query.getDinstinctValuesQuery(); - - SortedSet providersIds = new TreeSet<>(); - - logger.trace("Going to request the following query: {}", sql); - ResultSet resultSet = statement.executeQuery(sql); - - while (resultSet.next()) { - String id = resultSet.getString(1); - providersIds.add(id); + public SortedMap> getSpaceTimeSeries(Set dataTypes) throws Exception { + /* + SortedMap> sortedMap = new TreeMap<>(); + setRequestedRecords(AggregatedStorageStatusRecord.class); + for(String dataType : dataTypes) { + Filter filter = new Filter(StorageStatusRecord.DATA_TYPE, dataType); + if(filters == null) { + filters = new HashSet<>(); } + filters.add(filter); - return providersIds; - }finally { - connection.close(); - } + SortedMap timeSeries = getTimeSeries(); + sortedMap.put(filter, timeSeries); + + filters.remove(filter); + } + return sortedMap; + */ + return null; } - + @Override public List getUsageValueQuotaTotal(List listUsage) throws Exception { return null; } - @Override - public SortedMap> getSpaceTimeSeries(Class> clz, - TemporalConstraint temporalConstraint, List filters, List providersId) throws Exception { - return null; - } @Override public void close() throws Exception { diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java index 3e7bf64..caf70b8 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java @@ -5,7 +5,6 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.gcube.accounting.analytics.Filter; @@ -32,7 +31,7 @@ public class Query extends PostgreSQLQuery { private Set requestedTableField; protected TemporalConstraint temporalConstraint; - protected List filters; + protected Set filters; protected Set contexts; private String tableFieldToRequest; @@ -72,7 +71,7 @@ public class Query extends PostgreSQLQuery { this.temporalConstraint = temporalConstraint; } - public void setFilters(List filters) { + public void setFilters(Set filters) { this.filters = filters; } diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java index 0d29f80..dea9dbe 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java @@ -6,8 +6,10 @@ package org.gcube.accounting.analytics.persistence.postgresql; import java.util.ArrayList; import java.util.Calendar; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -25,6 +27,7 @@ import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; +import org.gcube.accounting.datamodel.basetypes.AbstractStorageUsageRecord.DataType; import org.gcube.accounting.utility.postgresql.RecordToDBMapping; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.Record; @@ -51,8 +54,8 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { accountingPersistenceQueryPostgreSQL.prepareConnection(configuration); } - protected List getFilters(){ - List filters = new ArrayList<>(); + protected Set getFilters(){ + Set filters = new HashSet<>(); String user = QueryTest.getRandomUser(); logger.debug("Going to query filtering with user {}", user); Filter filter = new Filter(UsageRecord.CONSUMER_ID, user); @@ -92,7 +95,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { @Test public void testGetNoContextTimeSeries() throws Exception { - List filters = getFilters(); + Set filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); @@ -102,7 +105,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); - SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getNoContextTimeSeries(clz, temporalConstraint, filters); + accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); @@ -112,7 +119,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { @Test public void testTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - List filters = getFilters(); + Set filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); @@ -123,7 +130,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); - SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(clz, temporalConstraint, filters); + accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); @@ -136,21 +147,27 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { @Test public void testContextTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - List filters = getFilters(); + Set filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); List>> classes = getClassesToTest(); - List contexts = new ArrayList<>(); + Set contexts = new HashSet<>(); contexts.add("/gcube/devsec/devVRE"); contexts.add("/gcube/devNext/NextNext"); for(Class> clz : classes) { logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); - - SortedMap> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries(clz, temporalConstraint, filters, contexts); + + accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setContexts(contexts); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + + SortedMap> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries(); for(Filter f : contextTimeseries.keySet()) { logger.debug("{}", f); SortedMap timeseries = contextTimeseries.get(f); @@ -167,7 +184,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { @Test public void testGetFilterValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - List filters = getFilters(); + Set filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); @@ -184,13 +201,17 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { String key = keysToRequest.get(clz); - SortedSet numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key); + accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + SortedSet numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key); for(NumberedFilter numberedFilter : numberedFilters) { logger.debug("{}", numberedFilter); } logger.debug("----Only first 3 results"); - numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key, 3); + numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3); for(NumberedFilter numberedFilter : numberedFilters) { logger.debug("{}", numberedFilter); } @@ -201,13 +222,17 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { public void testTopValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { String orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(AggregatedServiceUsageRecord.class); - List filters = new ArrayList<>(); + Set filters = new HashSet<>(); Filter filter = new Filter(UsageRecord.CONSUMER_ID, QueryTest.getRandomUser()); filters.add(filter); TemporalConstraint temporalConstraint = getTemporalConstraint(); - SortedMap> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty); + accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedServiceUsageRecord.class); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + SortedMap> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty); for(NumberedFilter numberedFilter : topTimeSeries.keySet()) { logger.debug("{}", numberedFilter); SortedMap timeseries = topTimeSeries.get(numberedFilter); @@ -226,8 +251,23 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { } @Test - public void testGetSpaceProvidersIds() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - SortedSet spaceProvidersIds = accountingPersistenceQueryPostgreSQL.getSpaceProvidersIds(); - logger.debug("{}", spaceProvidersIds); + public void testGetSpaceTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { + + TemporalConstraint temporalConstraint = getTemporalConstraint(); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + + Set dataTypes = new HashSet<>(); + dataTypes.add(DataType.STORAGE.name()); + + SortedMap> getSpaceTimeSeries = accountingPersistenceQueryPostgreSQL.getSpaceTimeSeries(dataTypes); + for(Filter filter : getSpaceTimeSeries.keySet()) { + logger.debug("{}", filter); + SortedMap timeseries = getSpaceTimeSeries.get(filter); + for(Calendar c : timeseries.keySet()) { + Info info = timeseries.get(c); + logger.debug("{}", info); + } + } } + } \ No newline at end of file diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java index 3abf398..699f43b 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java @@ -3,10 +3,8 @@ */ package org.gcube.accounting.analytics.persistence.postgresql; -import java.util.ArrayList; import java.util.Calendar; import java.util.HashSet; -import java.util.List; import java.util.Random; import java.util.Set; @@ -57,7 +55,7 @@ public class QueryTest extends ContextTest { entTimeCalendar.set(Calendar.HOUR_OF_DAY, 16); entTimeCalendar.set(Calendar.MINUTE, 17); - List filters = new ArrayList<>(); + Set filters = new HashSet<>(); Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser()); filters.add(filter); From 16da114f8b421e98a2ff561d8ffe17b44f9f15d2 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Wed, 28 Jul 2021 13:04:27 +0200 Subject: [PATCH 2/5] backported patch --- .../postgresql/AccountingPersistenceQueryPostgreSQL.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java index 0acaa51..eb608bb 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java @@ -194,7 +194,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten SortedMap> ret = new TreeMap<>(); for(String context : contexts) { - Filter contextFilter = new Filter("context", context); + Filter contextFilter = new Filter("scope", context); Set ctxs = new HashSet<>(); ctxs.add(context); From 3657057e48009ead695bb7c51414f86de376f465 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Fri, 29 Oct 2021 19:02:49 +0200 Subject: [PATCH 3/5] Added the possibility of having OR conditions between filters with the same key --- .../persistence/postgresql/Query.java | 64 +++++++++---------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java index caf70b8..c54ba33 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java @@ -2,9 +2,12 @@ package org.gcube.accounting.analytics.persistence.postgresql; import java.sql.SQLException; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import org.gcube.accounting.analytics.Filter; @@ -31,7 +34,7 @@ public class Query extends PostgreSQLQuery { private Set requestedTableField; protected TemporalConstraint temporalConstraint; - protected Set filters; + protected Map> filters; protected Set contexts; private String tableFieldToRequest; @@ -40,24 +43,6 @@ public class Query extends PostgreSQLQuery { private String recordId; - protected String orConditionKey; - protected Set orConditionValues; - - public void setOrConditionKey(String orConditionKey) { - this.orConditionKey = orConditionKey; - } - - public void setOrConditionValues(Collection orConditionValues) { - this.orConditionValues = new HashSet<>(orConditionValues); - } - - public void addOrConditionValue(String conditionValue) { - if(this.orConditionValues == null) { - this.orConditionValues = new HashSet<>(); - } - this.orConditionValues.add(conditionValue); - } - public Query(Class> clz) throws Exception { this.clz = clz; this.recordToDBFields = RecordToDBMapping.getRecordToDBFields(clz); @@ -71,8 +56,20 @@ public class Query extends PostgreSQLQuery { this.temporalConstraint = temporalConstraint; } - public void setFilters(Set filters) { - this.filters = filters; + public void setFilters(Collection filters) { + this.filters = new HashMap<>(); + if(filters!=null && filters.size()>0) { + for(Filter filter : filters) { + String key = filter.getKey(); + String value = filter.getValue(); + Collection list = this.filters.get(key); + if(list == null) { + list = new ArrayList<>(); + this.filters.put(key, list); + } + list.add(value); + } + } } public void setContexts(Set contexts) { @@ -120,28 +117,26 @@ public class Query extends PostgreSQLQuery { return date; } + //TODO need to check duplicated keys and put all values in OR protected void addFilters() throws SQLException { if(filters!=null && filters.size()>0) { - // The first filter if the time_bucket - stringBuffer.append(" AND "); + stringBuffer.append("("); boolean first = true; - for(Filter filter : filters) { + for(String key : filters.keySet()) { if(first) { first = false; }else { stringBuffer.append(" AND "); } - appendTableField(filter.getKey()); - stringBuffer.append("="); - appendValue(filter.getValue()); + addOrConditions(key, filters.get(key)); } + stringBuffer.append(")"); } } - protected void addOrConditions(String key, Set values) throws SQLException { + protected void addOrConditions(String key, Collection values) throws SQLException { if(values!=null && values.size()>0) { - // The first filter if the time_bucket - stringBuffer.append(" AND ("); + stringBuffer.append("("); boolean first = true; for(String value : values) { if(first) { @@ -157,15 +152,10 @@ public class Query extends PostgreSQLQuery { } } - protected void addContextFilter() throws SQLException { addOrConditions(UsageRecord.SCOPE, contexts); } - protected void addOrConditionFilter() throws SQLException { - addOrConditions(orConditionKey, orConditionValues); - } - protected void addEmittedFields(boolean addNested) throws Exception { Set aggregatedField = clz.newInstance().getAggregatedFields(); for(String fieldName : aggregatedField) { @@ -357,7 +347,9 @@ public class Query extends PostgreSQLQuery { addTemporalConstraintToQuery(); + stringBuffer.append(" AND "); addFilters(); + stringBuffer.append(" AND "); addContextFilter(); addDateGroupBy(false); @@ -384,7 +376,9 @@ public class Query extends PostgreSQLQuery { addTemporalConstraintToQuery(); + stringBuffer.append(" AND "); addFilters(); + stringBuffer.append(" AND "); addContextFilter(); addGroupAndOrderByForOrderByField(); From a1133909330ac11da67202f5f33e0aa7fb8aae38 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 2 Nov 2021 11:29:29 +0100 Subject: [PATCH 4/5] Validated query builder --- .../persistence/postgresql/Query.java | 20 +++++++++++++------ .../persistence/postgresql/QueryTest.java | 9 +++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java index c54ba33..8955213 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java @@ -117,10 +117,11 @@ public class Query extends PostgreSQLQuery { return date; } - //TODO need to check duplicated keys and put all values in OR protected void addFilters() throws SQLException { if(filters!=null && filters.size()>0) { - stringBuffer.append("("); + if(filters.size()>1) { + stringBuffer.append("("); + } boolean first = true; for(String key : filters.keySet()) { if(first) { @@ -130,13 +131,17 @@ public class Query extends PostgreSQLQuery { } addOrConditions(key, filters.get(key)); } - stringBuffer.append(")"); + if(filters.size()>1) { + stringBuffer.append(")"); + } } } protected void addOrConditions(String key, Collection values) throws SQLException { if(values!=null && values.size()>0) { - stringBuffer.append("("); + if(values.size()>1) { + stringBuffer.append("("); + } boolean first = true; for(String value : values) { if(first) { @@ -148,7 +153,9 @@ public class Query extends PostgreSQLQuery { stringBuffer.append("="); appendValue(value); } - stringBuffer.append(")"); + if(values.size()>1) { + stringBuffer.append(")"); + } } } @@ -251,7 +258,7 @@ public class Query extends PostgreSQLQuery { } protected void addTemporalConstraintToQuery() throws SQLException { - stringBuffer.append(" WHERE "); + stringBuffer.append(" WHERE ("); String tableField = getTableField(AggregatedRecord.START_TIME); stringBuffer.append(tableField); stringBuffer.append(" > "); @@ -262,6 +269,7 @@ public class Query extends PostgreSQLQuery { stringBuffer.append(tableField); stringBuffer.append(" < "); appendValue(temporalConstraint.getAlignedEndTime()); + stringBuffer.append(")"); } protected void addDateGroupBy(boolean addNested) { diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java index 699f43b..fca5c4f 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java @@ -11,7 +11,6 @@ import java.util.Set; import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; -import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; @@ -58,6 +57,8 @@ public class QueryTest extends ContextTest { Set filters = new HashSet<>(); Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser()); filters.add(filter); + filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser()); + filters.add(filter); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY); Query query = new Query(AggregatedServiceUsageRecord.class); @@ -67,9 +68,9 @@ public class QueryTest extends ContextTest { logger.debug(ret); - String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); Set contexts = new HashSet<>(); - contexts.add(currentScope); + contexts.add(DEFAULT_TEST_SCOPE); + contexts.add(ALTERNATIVE_TEST_SCOPE); query.setContexts(contexts); ret = query.getTimeSeriesQuery(); logger.debug(ret); @@ -89,7 +90,7 @@ public class QueryTest extends ContextTest { @Test public void testGetRecordQuery() throws Exception { Query query = new Query(AggregatedServiceUsageRecord.class); - query.setRecordId("fa573711-ceb6-44ba-9c83-bd47e0915b80"); + query.setRecordId("7c3fcb31-6909-451d-bae0-633b3bec0c21"); String ret = query.getRecordQuery(); logger.debug(ret); } From 6aa36d7ad289720f4be65e0b039b99eba00def7f Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 2 Nov 2021 11:31:26 +0100 Subject: [PATCH 5/5] Fixed changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ff17f..00aa26f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for Accounting Analytics Backend Connector for PostgreSQL -## [v1.1.0-SNAPSHOT] +## [v2.0.0-SNAPSHOT] + +- Added possibility of having multiple values for the same key which will be used in OR [#21353] -- ## [v1.0.0]