diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java index 5cd4994..b300d3e 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java @@ -15,6 +15,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; +import java.util.TreeSet; import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Info; @@ -26,11 +27,12 @@ import org.gcube.accounting.analytics.exception.KeyException; import org.gcube.accounting.analytics.exception.ValueException; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; +import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; -import org.gcube.accounting.utility.postgresql.RecordToDBFields; import org.gcube.accounting.utility.postgresql.RecordToDBConnection; +import org.gcube.accounting.utility.postgresql.RecordToDBFields; import org.gcube.accounting.utility.postgresql.RecordToDBMapping; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.RecordUtility; @@ -47,6 +49,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z"; + public static final String URL_PROPERTY_KEY = AccountingPersistenceConfiguration.URL_PROPERTY_KEY; // private String baseURL; @@ -81,7 +84,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten } } - public SortedMap getTimeSeries(Class> clz, + protected SortedMap getTimeSeries(Class> clz, TemporalConstraint temporalConstraint, List filters, Set contexts) throws Exception { Connection connection = getConnection(clz); @@ -148,6 +151,84 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten return getTimeSeries(clz, temporalConstraint, filters, null); } + protected SortedSet getNumberedValues(Class> clz, + TemporalConstraint temporalConstraint, List filters, String key, + String orderingProperty, Integer limit) throws Exception { + Connection connection = getConnection(clz); + try { + Statement statement = connection.createStatement(); + + if(orderingProperty == null) { + orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(clz); + } + + if(limit == null) { + limit = 50; + } + + SortedSet result = new TreeSet<>(); + + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); + Set contexts = new HashSet<>(); + contexts.add(currentScope); + + Query query = new Query(clz); + query.setTemporalConstraint(temporalConstraint); + query.setFilters(filters); + query.setContexts(contexts); + query.setFieldOfRequesteValues(key); + query.setOrderByField(orderingProperty); + query.setLimit(limit); + + String sql = query.getNextPossibleValueQuery(); + + // List requestedTableField = query.getRequestedTableField(); + RecordToDBFields recordToDBMapper = query.getRecordToDBMapper(); + + ResultSet resultSet = statement.executeQuery(sql); + + while (resultSet.next()) { + String tableFieldKey = recordToDBMapper.getTableField(key); + Object value = resultSet.getObject(tableFieldKey); + Object numberObject = resultSet.getObject(orderingProperty); + NumberedFilter numberedFilter = new NumberedFilter(key, value.toString(), (Number) numberObject, orderingProperty); + result.add(numberedFilter); + } + return result; + + }finally { + connection.close(); + } + } + + @Override + public SortedSet getFilterValues(Class> clz, + TemporalConstraint temporalConstraint, List filters, String key) throws Exception { + return getNumberedValues(clz, temporalConstraint, filters, key, null, null); + } + + @Override + public SortedSet getFilterValues(Class> clz, + TemporalConstraint temporalConstraint, List filters, String key, Integer limit) throws Exception { + return getNumberedValues(clz, temporalConstraint, filters, key, null, limit); + } + + @Override + @Deprecated + public SortedSet getNextPossibleValues(Class> clz, + TemporalConstraint temporalConstraint, List filters, String key, String orderingProperty) + throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { + return getNumberedValues(clz, temporalConstraint, filters, key, orderingProperty, null); + } + + @Override + @Deprecated + public SortedSet getNextPossibleValuesWithMap(Class> clz, + TemporalConstraint temporalConstraint, List filters, String key, String orderingProperty) + throws Exception { + return getNumberedValues(clz, temporalConstraint, filters, key, orderingProperty, null); + } + @Override public SortedMap> getTopValues( Class> clz, TemporalConstraint temporalConstraint, List filters, @@ -157,41 +238,6 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten return null; } - @Override - public SortedSet getFilterValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key) throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public SortedSet getNextPossibleValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key, String orderingProperty) - throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public SortedSet getNextPossibleValuesWithMap(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key, String orderingProperty) - throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public void close() throws Exception { - // OK - } - - @Override - public SortedSet getFilterValues(Class> clz, - TemporalConstraint temporalConstraint, List filters, String key, Integer limit) throws Exception { - // TODO Auto-generated method stub - return null; - } - @Override public JSONObject getUsageValue(Class> clz, TemporalConstraint temporalConstraint, Filter applicant) throws Exception { @@ -232,10 +278,14 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten return null; } + @Override + public void close() throws Exception { + // OK + } + @Override public boolean isConnectionActive() throws Exception { return true; } - } diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java index fef2b2c..97d9661 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java @@ -30,6 +30,11 @@ public class Query extends PostgreSQLQuery { protected TemporalConstraint temporalConstraint; protected List filters; protected Set contexts; + + private String fieldOfRequesteValues; + private String orderByField; + private Integer limit; + public Query(Class> clz) throws Exception { this.clz = clz; @@ -52,6 +57,18 @@ public class Query extends PostgreSQLQuery { this.contexts = contexts; } + public void setFieldOfRequesteValues(String fieldOfRequesteValues) { + this.fieldOfRequesteValues = fieldOfRequesteValues; + } + + public void setOrderByField(String orderByField) { + this.orderByField = orderByField; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + public void addContext(String context) { if(contexts == null) { contexts = new HashSet<>(); @@ -115,7 +132,6 @@ public class Query extends PostgreSQLQuery { } } - protected void addEmittedFields() throws Exception { Set aggregatedField = clz.newInstance().getAggregatedFields(); for(String fieldName : aggregatedField) { @@ -126,7 +142,6 @@ public class Query extends PostgreSQLQuery { case AggregatedRecord.AGGREGATED: continue; - case AggregatedRecord.OPERATION_COUNT: stringBuffer.append(", SUM("); stringBuffer.append(dbField); @@ -171,13 +186,13 @@ public class Query extends PostgreSQLQuery { return calendarEnum.name().toLowerCase(); } - protected void addTimeBucket() { - stringBuffer.append("time_bucket('1 "); + protected void addRequestedDate() { + stringBuffer.append("date_trunc('"); String calendarInterval = getTimeBucketCalendarInterval(temporalConstraint.getAggregationMode()); stringBuffer.append(calendarInterval); - stringBuffer.append("',"); + stringBuffer.append("', "); appendTableField(AggregatedRecord.START_TIME); - stringBuffer.append(") AS "); + stringBuffer.append(") "); stringBuffer.append(DATE_OF_TIMESERIES_AS_FIELD); requestedTableField.add(DATE_OF_TIMESERIES_AS_FIELD); } @@ -201,21 +216,52 @@ public class Query extends PostgreSQLQuery { appendValue(temporalConstraint.getAlignedEndTime()); } - protected void addGropuBy() { + protected void addDateGropuBy() { stringBuffer.append(" GROUP BY "); stringBuffer.append(DATE_OF_TIMESERIES_AS_FIELD); } - protected void addOrderBy() { + protected void addDateOrderBy() { stringBuffer.append(" ORDER BY "); stringBuffer.append(DATE_OF_TIMESERIES_AS_FIELD); stringBuffer.append(" ASC"); } + protected void addGroupAndOrderByForOrderByField() { + stringBuffer.append(" GROUP BY "); + String dbField = getTableField(fieldOfRequesteValues); + stringBuffer.append(dbField); + stringBuffer.append(" ORDER BY "); + stringBuffer.append(orderByField); + stringBuffer.append(" DESC"); + } + + protected void addLimit() { + stringBuffer.append(" LIMIT "); + if(limit != null) { + stringBuffer.append(limit.toString()); + }else { + stringBuffer.append(50); + } + } + + protected void addRequestedField() { + String dbField = getTableField(fieldOfRequesteValues); + stringBuffer.append(dbField); + requestedTableField.add(dbField); + stringBuffer.append(", SUM("); + dbField = getTableField(orderByField); + stringBuffer.append(dbField); + requestedTableField.add(dbField); + stringBuffer.append(") AS "); + stringBuffer.append(orderByField); + + } + public String getTimeSeriesQuery() throws Exception { newQuery(); - addTimeBucket(); + addRequestedDate(); addEmittedFields(); @@ -227,8 +273,29 @@ public class Query extends PostgreSQLQuery { addFilters(); addContextFilter(); - addGropuBy(); - addOrderBy(); + addDateGropuBy(); + addDateOrderBy(); + + return stringBuffer.toString(); + } + + + + public String getNextPossibleValueQuery() throws Exception { + newQuery(); + + addRequestedField(); + + stringBuffer.append(" FROM "); + stringBuffer.append(recordToDBFields.getTableName()); + + addTemporalConstraintToQuery(); + + addFilters(); + addContextFilter(); + + addGroupAndOrderByForOrderByField(); + addLimit(); return stringBuffer.toString(); } diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java index 9f9ad63..194ceb3 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java @@ -7,9 +7,11 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.SortedMap; +import java.util.SortedSet; import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Info; +import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException; @@ -96,4 +98,41 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { } } + + @Test + public void testGetFilterValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { + Calendar startTimeCalendar = Calendar.getInstance(); + startTimeCalendar.set(Calendar.YEAR, 2020); + startTimeCalendar.set(Calendar.MONTH, Calendar.MARCH); + startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); + startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); + startTimeCalendar.set(Calendar.MINUTE, 0); + + Calendar entTimeCalendar = Calendar.getInstance(); + entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH); + entTimeCalendar.set(Calendar.DAY_OF_MONTH, 15); + entTimeCalendar.set(Calendar.HOUR_OF_DAY, 16); + entTimeCalendar.set(Calendar.MINUTE, 17); + + List filters = new ArrayList<>(); + Filter filter = new Filter(UsageRecord.CONSUMER_ID, QueryTest.getRandomUser()); + filters.add(filter); + + TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MINUTELY); + SortedSet numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD); + for(NumberedFilter numberedFilter : numberedFilters) { + logger.debug("{}", numberedFilter); + } + + numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD); + for(NumberedFilter numberedFilter : numberedFilters) { + logger.debug("{}", numberedFilter); + } + + + numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, 2); + for(NumberedFilter numberedFilter : numberedFilters) { + logger.debug("{}", numberedFilter); + } + } } \ No newline at end of file diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java index 67af507..cc35a9f 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java @@ -14,8 +14,10 @@ import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery; +import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,13 +61,14 @@ public class QueryTest extends ContextTest { Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser()); filters.add(filter); - TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MINUTELY); + TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY); Query query = new Query(AggregatedServiceUsageRecord.class); query.setTemporalConstraint(temporalConstraint); query.setFilters(filters); String ret = query.getTimeSeriesQuery(); logger.debug(ret); + String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery(); Set contexts = new HashSet<>(); contexts.add(currentScope); @@ -73,6 +76,16 @@ public class QueryTest extends ContextTest { ret = query.getTimeSeriesQuery(); logger.debug(ret); + + query = new Query(AggregatedServiceUsageRecord.class); + query.setTemporalConstraint(temporalConstraint); + query.setFilters(filters); + query.setContexts(contexts); + query.setFieldOfRequesteValues(ServiceUsageRecord.CALLED_METHOD); + query.setOrderByField(AccountingPersistenceQuery.getDefaultOrderingProperties(AggregatedServiceUsageRecord.class)); + ret = query.getNextPossibleValueQuery(); + logger.debug(ret); + } }