diff --git a/CHANGELOG.md b/CHANGELOG.md index c82d990..b3ff17f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for Accounting Analytics Backend Connector for PostgreSQL +## [v1.1.0-SNAPSHOT] + +- ## [v1.0.0] diff --git a/pom.xml b/pom.xml index 83659d4..387eb4c 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.accounting accounting-analytics-persistence-postgresql - 1.0.0 + 2.0.0-SNAPSHOT Accounting Analytics Backend Connector for PostgreSQL Accounting Analytics Backend Connector for PostgreSQL @@ -44,7 +44,7 @@ org.gcube.accounting accounting-analytics - [3.0.0, 4.0.0-SNAPSHOT) + [4.0.0-SNAPSHOT, 5.0.0) org.gcube.accounting diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java index 3f87586..0acaa51 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java @@ -23,23 +23,20 @@ import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.UsageValue; -import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException; -import org.gcube.accounting.analytics.exception.KeyException; -import org.gcube.accounting.analytics.exception.ValueException; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; -import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; import org.gcube.accounting.utility.postgresql.RecordToDBConnection; import org.gcube.accounting.utility.postgresql.RecordToDBFields; import org.gcube.accounting.utility.postgresql.RecordToDBMapping; +import org.gcube.com.fasterxml.jackson.databind.ObjectMapper; +import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.RecordUtility; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +55,39 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten protected AccountingPersistenceBackendQueryConfiguration configuration; + protected ObjectMapper objectMapper; + + protected Class> clz; + protected TemporalConstraint temporalConstraint; + + protected Set contexts; + protected Set filters; + + + public AccountingPersistenceQueryPostgreSQL() { + objectMapper = new ObjectMapper(); + } + + @Override + public void setRequestedRecords(Class> clz) { + this.clz = clz; + } + + @Override + public void setTemporalConstraint(TemporalConstraint temporalConstraint) { + this.temporalConstraint = temporalConstraint; + } + + @Override + public void setContexts(Set contexts) { + this.contexts = contexts; + } + + @Override + public void setFilters(Set filters) { + this.filters = filters; + } + static { // One Record per package is enough RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage()); @@ -87,9 +117,24 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } } - protected SortedMap getTimeSeries(Class> clz, - TemporalConstraint temporalConstraint, List filters, Set contexts) - throws Exception { + protected void addProperty(ObjectNode objectNode, String key, Object value) { + if(value instanceof Number) { + + if(value instanceof Integer) { + objectNode.put(key, (int) value); + return; + } + + Long longValue = Long.valueOf(value.toString()); + objectNode.put(key, longValue); + return; + } + + objectNode.put(key, (String) value.toString()); + + } + + protected SortedMap getTimeSeries(Set contexts) throws Exception { Connection connection = getConnection(clz); try { Statement statement = connection.createStatement(); @@ -114,15 +159,15 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten OffsetDateTime offsetDateTime = resultSet.getObject(tableFieldName, OffsetDateTime.class); Calendar calendar = getCalendar(offsetDateTime); - JSONObject jsonObject = new JSONObject(); + ObjectNode objectNode = objectMapper.createObjectNode(); for(String tableField : requestedTableField) { String usageRecordField = recordToDBMapper.getRecordField(tableField); - Object object = resultSet.getObject(tableField); - jsonObject.put(usageRecordField, object); + Object obj = resultSet.getObject(tableField); + addProperty(objectNode, usageRecordField, obj); } - Info info = new Info(calendar, jsonObject); + Info info = new Info(calendar, objectNode); result.put(calendar, info); } @@ -133,13 +178,8 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } @Override - public SortedMap getTimeSeries(Class> clz, - TemporalConstraint temporalConstraint, List filters) - throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - String context = AccountingPersistenceBackendQuery.getScopeToQuery(); - Set contexts = new HashSet<>(); - contexts.add(context); - return getTimeSeries(clz, temporalConstraint, filters, contexts); + public SortedMap getTimeSeries() throws Exception { + return getTimeSeries(contexts); } protected Calendar getCalendar(OffsetDateTime offsetDateTime) { @@ -148,25 +188,17 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten calendar.setTimeInMillis(epochMillis); return calendar; } - - @Override - public SortedMap getNoContextTimeSeries(Class> clz, - TemporalConstraint temporalConstraint, List filters) - throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - return getTimeSeries(clz, temporalConstraint, filters, null); - } @Override - public SortedMap> getContextTimeSeries( - Class> clz, TemporalConstraint temporalConstraint, List filters, - List contexts) throws Exception { + public SortedMap> getContextTimeSeries() throws Exception { SortedMap> ret = new TreeMap<>(); for(String context : contexts) { Filter contextFilter = new Filter("context", context); - Set timeSeriesContexts = new HashSet<>(); - timeSeriesContexts.add(context); - SortedMap timeSeries = getTimeSeries(clz, temporalConstraint, filters, timeSeriesContexts); + Set ctxs = new HashSet<>(); + ctxs.add(context); + + SortedMap timeSeries = getTimeSeries(ctxs); if(!timeSeries.isEmpty()) { ret.put(contextFilter, timeSeries); } @@ -174,9 +206,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten return ret; } - protected SortedSet getNumberedValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key, - String orderingProperty, Integer limit) throws Exception { + protected SortedSet getNumberedValues(String key, String orderingProperty, Integer limit) throws Exception { Connection connection = getConnection(clz); try { Statement statement = connection.createStatement(); @@ -191,10 +221,6 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten SortedSet result = new TreeSet<>(); - String context = AccountingPersistenceBackendQuery.getScopeToQuery(); - Set contexts = new HashSet<>(); - contexts.add(context); - Query query = new Query(clz); query.setTemporalConstraint(temporalConstraint); query.setFilters(filters); @@ -229,34 +255,26 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } @Override - public SortedSet getFilterValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key) throws Exception { - return getNumberedValues(clz, temporalConstraint, filters, key, null, null); + public SortedSet getFilterValues(String key) throws Exception { + return getNumberedValues(key, null, null); } @Override - public SortedSet getFilterValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key, Integer limit) throws Exception { - return getNumberedValues(clz, temporalConstraint, filters, key, null, limit); + public SortedSet getFilterValues(String key, Integer limit) throws Exception { + return getNumberedValues(key, null, limit); } @Override - public SortedMap> getTopValues( - Class> clz, TemporalConstraint temporalConstraint, List filters, - String topKey, String orderingProperty) - throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - - String context = AccountingPersistenceBackendQuery.getScopeToQuery(); - Set contexts = new HashSet<>(); - contexts.add(context); + public SortedMap> getTopValues(String topKey, String orderingProperty) + throws Exception { SortedMap> ret = new TreeMap<>(); - SortedSet top = getNumberedValues(clz, temporalConstraint, filters, topKey, orderingProperty, 10); + SortedSet top = getNumberedValues(topKey, orderingProperty, 10); for(NumberedFilter numberedFilter : top) { filters.add(numberedFilter); - SortedMap map = getTimeSeries(clz, temporalConstraint, filters, contexts); + SortedMap map = getTimeSeries(); ret.put(numberedFilter, map); filters.remove(numberedFilter); } @@ -309,42 +327,32 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } @Override - public SortedSet getSpaceProvidersIds() throws Exception { - Class> aggregatedRecordClass = AggregatedStorageStatusRecord.class; - Connection connection = getConnection(aggregatedRecordClass); - try { - Statement statement = connection.createStatement(); - - Query query = new Query(aggregatedRecordClass); - query.setTableFieldToRequest(AggregatedStorageStatusRecord.PROVIDER_ID); - String sql = query.getDinstinctValuesQuery(); - - SortedSet providersIds = new TreeSet<>(); - - logger.trace("Going to request the following query: {}", sql); - ResultSet resultSet = statement.executeQuery(sql); - - while (resultSet.next()) { - String id = resultSet.getString(1); - providersIds.add(id); + public SortedMap> getSpaceTimeSeries(Set dataTypes) throws Exception { + /* + SortedMap> sortedMap = new TreeMap<>(); + setRequestedRecords(AggregatedStorageStatusRecord.class); + for(String dataType : dataTypes) { + Filter filter = new Filter(StorageStatusRecord.DATA_TYPE, dataType); + if(filters == null) { + filters = new HashSet<>(); } + filters.add(filter); - return providersIds; - }finally { - connection.close(); - } + SortedMap timeSeries = getTimeSeries(); + sortedMap.put(filter, timeSeries); + + filters.remove(filter); + } + return sortedMap; + */ + return null; } - + @Override public List getUsageValueQuotaTotal(List listUsage) throws Exception { return null; } - @Override - public SortedMap> getSpaceTimeSeries(Class> clz, - TemporalConstraint temporalConstraint, List filters, List providersId) throws Exception { - return null; - } @Override public void close() throws Exception { diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java index 3e7bf64..caf70b8 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java @@ -5,7 +5,6 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.gcube.accounting.analytics.Filter; @@ -32,7 +31,7 @@ public class Query extends PostgreSQLQuery { private Set requestedTableField; protected TemporalConstraint temporalConstraint; - protected List filters; + protected Set filters; protected Set contexts; private String tableFieldToRequest; @@ -72,7 +71,7 @@ public class Query extends PostgreSQLQuery { this.temporalConstraint = temporalConstraint; } - public void setFilters(List filters) { + public void setFilters(Set filters) { this.filters = filters; } diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java index 0d29f80..dea9dbe 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java @@ -6,8 +6,10 @@ package org.gcube.accounting.analytics.persistence.postgresql; import java.util.ArrayList; import java.util.Calendar; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -25,6 +27,7 @@ import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; +import org.gcube.accounting.datamodel.basetypes.AbstractStorageUsageRecord.DataType; import org.gcube.accounting.utility.postgresql.RecordToDBMapping; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.Record; @@ -51,8 +54,8 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { accountingPersistenceQueryPostgreSQL.prepareConnection(configuration); } - protected List getFilters(){ - List filters = new ArrayList<>(); + protected Set getFilters(){ + Set filters = new HashSet<>(); String user = QueryTest.getRandomUser(); logger.debug("Going to query filtering with user {}", user); Filter filter = new Filter(UsageRecord.CONSUMER_ID, user); @@ -92,7 +95,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { @Test public void testGetNoContextTimeSeries() throws Exception { - List filters = getFilters(); + Set filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); @@ -102,7 +105,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); - SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getNoContextTimeSeries(clz, temporalConstraint, filters); + accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); @@ -112,7 +119,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { @Test public void testTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - List filters = getFilters(); + Set filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); @@ -123,7 +130,11 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); - SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(clz, temporalConstraint, filters); + accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); @@ -136,21 +147,27 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { @Test public void testContextTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - List filters = getFilters(); + Set filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); List>> classes = getClassesToTest(); - List contexts = new ArrayList<>(); + Set contexts = new HashSet<>(); contexts.add("/gcube/devsec/devVRE"); contexts.add("/gcube/devNext/NextNext"); for(Class> clz : classes) { logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); - - SortedMap> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries(clz, temporalConstraint, filters, contexts); + + accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setContexts(contexts); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + + SortedMap> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries(); for(Filter f : contextTimeseries.keySet()) { logger.debug("{}", f); SortedMap timeseries = contextTimeseries.get(f); @@ -167,7 +184,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { @Test public void testGetFilterValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - List filters = getFilters(); + Set filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); @@ -184,13 +201,17 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { String key = keysToRequest.get(clz); - SortedSet numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key); + accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + SortedSet numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key); for(NumberedFilter numberedFilter : numberedFilters) { logger.debug("{}", numberedFilter); } logger.debug("----Only first 3 results"); - numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(clz, temporalConstraint, filters, key, 3); + numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3); for(NumberedFilter numberedFilter : numberedFilters) { logger.debug("{}", numberedFilter); } @@ -201,13 +222,17 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { public void testTopValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { String orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(AggregatedServiceUsageRecord.class); - List filters = new ArrayList<>(); + Set filters = new HashSet<>(); Filter filter = new Filter(UsageRecord.CONSUMER_ID, QueryTest.getRandomUser()); filters.add(filter); TemporalConstraint temporalConstraint = getTemporalConstraint(); - SortedMap> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty); + accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedServiceUsageRecord.class); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + accountingPersistenceQueryPostgreSQL.setFilters(filters); + + SortedMap> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty); for(NumberedFilter numberedFilter : topTimeSeries.keySet()) { logger.debug("{}", numberedFilter); SortedMap timeseries = topTimeSeries.get(numberedFilter); @@ -226,8 +251,23 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { } @Test - public void testGetSpaceProvidersIds() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - SortedSet spaceProvidersIds = accountingPersistenceQueryPostgreSQL.getSpaceProvidersIds(); - logger.debug("{}", spaceProvidersIds); + public void testGetSpaceTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { + + TemporalConstraint temporalConstraint = getTemporalConstraint(); + accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); + + Set dataTypes = new HashSet<>(); + dataTypes.add(DataType.STORAGE.name()); + + SortedMap> getSpaceTimeSeries = accountingPersistenceQueryPostgreSQL.getSpaceTimeSeries(dataTypes); + for(Filter filter : getSpaceTimeSeries.keySet()) { + logger.debug("{}", filter); + SortedMap timeseries = getSpaceTimeSeries.get(filter); + for(Calendar c : timeseries.keySet()) { + Info info = timeseries.get(c); + logger.debug("{}", info); + } + } } + } \ No newline at end of file diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java index 3abf398..699f43b 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java @@ -3,10 +3,8 @@ */ package org.gcube.accounting.analytics.persistence.postgresql; -import java.util.ArrayList; import java.util.Calendar; import java.util.HashSet; -import java.util.List; import java.util.Random; import java.util.Set; @@ -57,7 +55,7 @@ public class QueryTest extends ContextTest { entTimeCalendar.set(Calendar.HOUR_OF_DAY, 16); entTimeCalendar.set(Calendar.MINUTE, 17); - List filters = new ArrayList<>(); + Set filters = new HashSet<>(); Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser()); filters.add(filter);