From aae19295caa49d33f1cfffc9d594aea2542bc8f8 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Wed, 31 Mar 2021 17:49:49 +0200 Subject: [PATCH] Timeseries required a nested query --- .../AccountingPersistenceQueryPostgreSQL.java | 2 +- .../persistence/postgresql/Query.java | 100 ++++++++++++++---- ...ountingPersistenceQueryPostgreSQLTest.java | 4 +- 3 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java index 879050b..ac3fc82 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java @@ -103,7 +103,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten String sql = query.getTimeSeriesQuery(); - List requestedTableField = query.getRequestedTableField(); + Set requestedTableField = query.getRequestedTableField(); RecordToDBFields recordToDBMapper = query.getRecordToDBMapper(); ResultSet resultSet = statement.executeQuery(sql); diff --git a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java index cfe82a1..c82594e 100644 --- a/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java +++ b/src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java @@ -1,7 +1,6 @@ package org.gcube.accounting.analytics.persistence.postgresql; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.HashSet; @@ -24,10 +23,12 @@ public class Query extends PostgreSQLQuery { public static final String DATE_OF_TIMESERIES_AS_FIELD = AggregatedRecord.START_TIME; + public static final String NESTED_TABLE_NAME = "nested"; + private Class> clz; private final RecordToDBFields recordToDBFields; - private List requestedTableField; + private Set requestedTableField; protected TemporalConstraint temporalConstraint; protected List filters; @@ -62,7 +63,7 @@ public class Query extends PostgreSQLQuery { this.recordToDBFields = RecordToDBMapping.getRecordToDBFields(clz); } - public List getRequestedTableField() { + public Set getRequestedTableField() { return requestedTableField; } @@ -165,7 +166,7 @@ public class Query extends PostgreSQLQuery { addOrConditions(orConditionKey, orConditionValues); } - protected void addEmittedFields() throws Exception { + protected void addEmittedFields(boolean addNested) throws Exception { Set aggregatedField = clz.newInstance().getAggregatedFields(); for(String fieldName : aggregatedField) { String dbField = getTableField(fieldName); @@ -177,12 +178,20 @@ public class Query extends PostgreSQLQuery { case AggregatedRecord.OPERATION_COUNT: stringBuffer.append(", SUM("); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } stringBuffer.append(dbField); stringBuffer.append(") AS "); break; case AggregatedStorageUsageRecord.DATA_VOLUME: stringBuffer.append(", SUM("); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } stringBuffer.append(dbField); stringBuffer.append(") AS "); break; @@ -190,22 +199,42 @@ public class Query extends PostgreSQLQuery { // WEIGHTED AVERAGE case AggregatedServiceUsageRecord.DURATION: stringBuffer.append(", ROUND(SUM("); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } stringBuffer.append(dbField); stringBuffer.append("*"); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } appendTableField(AggregatedRecord.OPERATION_COUNT); stringBuffer.append(")/SUM("); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } appendTableField(AggregatedRecord.OPERATION_COUNT); stringBuffer.append(")) AS "); break; case AggregatedServiceUsageRecord.MAX_INVOCATION_TIME: stringBuffer.append(", MAX("); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } stringBuffer.append(dbField); stringBuffer.append(") AS "); break; case AggregatedServiceUsageRecord.MIN_INVOCATION_TIME: stringBuffer.append(", MIN("); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } stringBuffer.append(dbField); stringBuffer.append(") AS "); break; @@ -225,21 +254,9 @@ public class Query extends PostgreSQLQuery { return calendarEnum.name().toLowerCase(); } - protected void addRequestedDate() { - String calendarInterval = getTimeBucketCalendarInterval(temporalConstraint.getAggregationMode()); - String tableField = getTableField(DATE_OF_TIMESERIES_AS_FIELD); - stringBuffer.append("date_trunc('"); - stringBuffer.append(calendarInterval); - stringBuffer.append("', "); - stringBuffer.append(tableField); - stringBuffer.append(") "); - stringBuffer.append(tableField); - requestedTableField.add(tableField); - } - private void newQuery() { stringBuffer = new StringBuffer(); - requestedTableField = new ArrayList<>(); + requestedTableField = new HashSet<>(); stringBuffer.append("SELECT "); } @@ -257,13 +274,21 @@ public class Query extends PostgreSQLQuery { appendValue(temporalConstraint.getAlignedEndTime()); } - protected void addDateGroupBy() { + protected void addDateGroupBy(boolean addNested) { stringBuffer.append(" GROUP BY "); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } appendTableField(DATE_OF_TIMESERIES_AS_FIELD); } - protected void addDateOrderBy() { + protected void addDateOrderBy(boolean addNested) { stringBuffer.append(" ORDER BY "); + if(addNested) { + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + } appendTableField(DATE_OF_TIMESERIES_AS_FIELD); stringBuffer.append(" ASC"); } @@ -299,12 +324,33 @@ public class Query extends PostgreSQLQuery { stringBuffer.append(dbField); } + protected void addRequestedDate() { + String calendarInterval = getTimeBucketCalendarInterval(temporalConstraint.getAggregationMode()); + String tableField = getTableField(DATE_OF_TIMESERIES_AS_FIELD); + stringBuffer.append("date_trunc('"); + stringBuffer.append(calendarInterval); + stringBuffer.append("', "); + stringBuffer.append(tableField); + stringBuffer.append(" AT TIME ZONE 'UTC') "); + stringBuffer.append(tableField); + requestedTableField.add(tableField); + } + public String getTimeSeriesQuery() throws Exception { newQuery(); - addRequestedDate(); + stringBuffer.append(NESTED_TABLE_NAME); + stringBuffer.append("."); + String tableField = getTableField(DATE_OF_TIMESERIES_AS_FIELD); + stringBuffer.append(tableField); - addEmittedFields(); + addEmittedFields(true); + + /* Nested Query */ + stringBuffer.append(" FROM ( SELECT "); + + addRequestedDate(); + addEmittedFields(false); stringBuffer.append(" FROM "); stringBuffer.append(recordToDBFields.getTableName()); @@ -314,8 +360,16 @@ public class Query extends PostgreSQLQuery { addFilters(); addContextFilter(); - addDateGroupBy(); - addDateOrderBy(); + addDateGroupBy(false); + addDateOrderBy(false); + + stringBuffer.append(") AS "); + stringBuffer.append(NESTED_TABLE_NAME); + + /* Nested Query */ + + addDateGroupBy(true); + addDateOrderBy(true); return stringBuffer.toString(); } diff --git a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java index 0e6d6b7..8068d70 100644 --- a/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java +++ b/src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java @@ -63,7 +63,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { protected TemporalConstraint getTemporalConstraint() { Calendar startTimeCalendar = Calendar.getInstance(); startTimeCalendar.set(Calendar.YEAR, 2015); - startTimeCalendar.set(Calendar.MONTH, Calendar.MARCH); + startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); startTimeCalendar.set(Calendar.MINUTE, 0); @@ -75,7 +75,7 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); entTimeCalendar.set(Calendar.MINUTE, 0); - TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MINUTELY); + TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY); return temporalConstraint; }