gCubeSystem
/
accounting-analytics-persistence-postgresql
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implementing queries

This commit is contained in:
Luca Frosini 2021-03-19 15:55:06 +01:00
parent cc4427a12b
commit e33c1328bd
4 changed files with 219 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
src/main/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQL.java
View File

@ -15,6 +15,7 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info;
@ -26,11 +27,12 @@ import org.gcube.accounting.analytics.exception.KeyException;
import org.gcube.accounting.analytics.exception.ValueException;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.accounting.persistence.AccountingPersistenceConfiguration;
import org.gcube.accounting.utility.postgresql.RecordToDBFields;
import org.gcube.accounting.utility.postgresql.RecordToDBConnection;
import org.gcube.accounting.utility.postgresql.RecordToDBFields;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.RecordUtility;
@ -47,6 +49,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z";
public static final String URL_PROPERTY_KEY = AccountingPersistenceConfiguration.URL_PROPERTY_KEY;
// private String baseURL;
@ -81,7 +84,7 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
}
}
public SortedMap<Calendar, Info> getTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
protected SortedMap<Calendar, Info> getTimeSeries(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, Set<String> contexts)
throws Exception {
Connection connection = getConnection(clz);
@ -148,6 +151,84 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
return getTimeSeries(clz, temporalConstraint, filters, null);
}
protected SortedSet<NumberedFilter> getNumberedValues(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key,
String orderingProperty, Integer limit) throws Exception {
Connection connection = getConnection(clz);
try {
Statement statement = connection.createStatement();
if(orderingProperty == null) {
orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(clz);
}
if(limit == null) {
limit = 50;
}
SortedSet<NumberedFilter> result = new TreeSet<>();
String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>();
contexts.add(currentScope);
Query query = new Query(clz);
query.setTemporalConstraint(temporalConstraint);
query.setFilters(filters);
query.setContexts(contexts);
query.setFieldOfRequesteValues(key);
query.setOrderByField(orderingProperty);
query.setLimit(limit);
String sql = query.getNextPossibleValueQuery();
// List<String> requestedTableField = query.getRequestedTableField();
RecordToDBFields recordToDBMapper = query.getRecordToDBMapper();
ResultSet resultSet = statement.executeQuery(sql);
while (resultSet.next()) {
String tableFieldKey = recordToDBMapper.getTableField(key);
Object value = resultSet.getObject(tableFieldKey);
Object numberObject = resultSet.getObject(orderingProperty);
NumberedFilter numberedFilter = new NumberedFilter(key, value.toString(), (Number) numberObject, orderingProperty);
result.add(numberedFilter);
}
return result;
}finally {
connection.close();
}
}
@Override
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key) throws Exception {
return getNumberedValues(clz, temporalConstraint, filters, key, null, null);
}
@Override
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key, Integer limit) throws Exception {
return getNumberedValues(clz, temporalConstraint, filters, key, null, limit);
}
@Override
@Deprecated
public SortedSet<NumberedFilter> getNextPossibleValues(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key, String orderingProperty)
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
return getNumberedValues(clz, temporalConstraint, filters, key, orderingProperty, null);
}
@Override
@Deprecated
public SortedSet<NumberedFilter> getNextPossibleValuesWithMap(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key, String orderingProperty)
throws Exception {
return getNumberedValues(clz, temporalConstraint, filters, key, orderingProperty, null);
}
@Override
public SortedMap<NumberedFilter, SortedMap<Calendar, Info>> getTopValues(
Class<? extends AggregatedRecord<?, ?>> clz, TemporalConstraint temporalConstraint, List<Filter> filters,
@ -157,41 +238,6 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
return null;
}
@Override
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key) throws Exception {
// TODO Auto-generated method stub
return null;
}
@Override
public SortedSet<NumberedFilter> getNextPossibleValues(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key, String orderingProperty)
throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
// TODO Auto-generated method stub
return null;
}
@Override
public SortedSet<NumberedFilter> getNextPossibleValuesWithMap(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key, String orderingProperty)
throws Exception {
// TODO Auto-generated method stub
return null;
}
@Override
public void close() throws Exception {
// OK
}
@Override
public SortedSet<NumberedFilter> getFilterValues(Class<? extends AggregatedRecord<?, ?>> clz,
TemporalConstraint temporalConstraint, List<Filter> filters, String key, Integer limit) throws Exception {
// TODO Auto-generated method stub
return null;
}
@Override
public JSONObject getUsageValue(Class<? extends AggregatedRecord<?, ?>> clz, TemporalConstraint temporalConstraint,
Filter applicant) throws Exception {
@ -232,10 +278,14 @@ public class AccountingPersistenceQueryPostgreSQL implements AccountingPersisten
return null;
}
@Override
public void close() throws Exception {
// OK
}
@Override
public boolean isConnectionActive() throws Exception {
return true;
}
}

89
src/main/java/org/gcube/accounting/analytics/persistence/postgresql/Query.java
View File

@ -30,6 +30,11 @@ public class Query extends PostgreSQLQuery {
protected TemporalConstraint temporalConstraint;
protected List<Filter> filters;
protected Set<String> contexts;
private String fieldOfRequesteValues;
private String orderByField;
private Integer limit;
public Query(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
this.clz = clz;
@ -52,6 +57,18 @@ public class Query extends PostgreSQLQuery {
this.contexts = contexts;
}
public void setFieldOfRequesteValues(String fieldOfRequesteValues) {
this.fieldOfRequesteValues = fieldOfRequesteValues;
}
public void setOrderByField(String orderByField) {
this.orderByField = orderByField;
}
public void setLimit(Integer limit) {
this.limit = limit;
}
public void addContext(String context) {
if(contexts == null) {
contexts = new HashSet<>();
@ -115,7 +132,6 @@ public class Query extends PostgreSQLQuery {
}
}
protected void addEmittedFields() throws Exception {
Set<String> aggregatedField = clz.newInstance().getAggregatedFields();
for(String fieldName : aggregatedField) {
@ -126,7 +142,6 @@ public class Query extends PostgreSQLQuery {
case AggregatedRecord.AGGREGATED:
continue;
case AggregatedRecord.OPERATION_COUNT:
stringBuffer.append(", SUM(");
stringBuffer.append(dbField);
@ -171,13 +186,13 @@ public class Query extends PostgreSQLQuery {
return calendarEnum.name().toLowerCase();
}
protected void addTimeBucket() {
stringBuffer.append("time_bucket('1 ");
protected void addRequestedDate() {
stringBuffer.append("date_trunc('");
String calendarInterval = getTimeBucketCalendarInterval(temporalConstraint.getAggregationMode());
stringBuffer.append(calendarInterval);
stringBuffer.append("',");
stringBuffer.append("', ");
appendTableField(AggregatedRecord.START_TIME);
stringBuffer.append(") AS ");
stringBuffer.append(") ");
stringBuffer.append(DATE_OF_TIMESERIES_AS_FIELD);
requestedTableField.add(DATE_OF_TIMESERIES_AS_FIELD);
}
@ -201,21 +216,52 @@ public class Query extends PostgreSQLQuery {
appendValue(temporalConstraint.getAlignedEndTime());
}
protected void addGropuBy() {
protected void addDateGropuBy() {
stringBuffer.append(" GROUP BY ");
stringBuffer.append(DATE_OF_TIMESERIES_AS_FIELD);
}
protected void addOrderBy() {
protected void addDateOrderBy() {
stringBuffer.append(" ORDER BY ");
stringBuffer.append(DATE_OF_TIMESERIES_AS_FIELD);
stringBuffer.append(" ASC");
}
protected void addGroupAndOrderByForOrderByField() {
stringBuffer.append(" GROUP BY ");
String dbField = getTableField(fieldOfRequesteValues);
stringBuffer.append(dbField);
stringBuffer.append(" ORDER BY ");
stringBuffer.append(orderByField);
stringBuffer.append(" DESC");
}
protected void addLimit() {
stringBuffer.append(" LIMIT ");
if(limit != null) {
stringBuffer.append(limit.toString());
}else {
stringBuffer.append(50);
}
}
protected void addRequestedField() {
String dbField = getTableField(fieldOfRequesteValues);
stringBuffer.append(dbField);
requestedTableField.add(dbField);
stringBuffer.append(", SUM(");
dbField = getTableField(orderByField);
stringBuffer.append(dbField);
requestedTableField.add(dbField);
stringBuffer.append(") AS ");
stringBuffer.append(orderByField);
}
public String getTimeSeriesQuery() throws Exception {
newQuery();
addTimeBucket();
addRequestedDate();
addEmittedFields();
@ -227,8 +273,29 @@ public class Query extends PostgreSQLQuery {
addFilters();
addContextFilter();
addGropuBy();
addOrderBy();
addDateGropuBy();
addDateOrderBy();
return stringBuffer.toString();
}
public String getNextPossibleValueQuery() throws Exception {
newQuery();
addRequestedField();
stringBuffer.append(" FROM ");
stringBuffer.append(recordToDBFields.getTableName());
addTemporalConstraintToQuery();
addFilters();
addContextFilter();
addGroupAndOrderByForOrderByField();
addLimit();
return stringBuffer.toString();
}

39
src/test/java/org/gcube/accounting/analytics/persistence/postgresql/AccountingPersistenceQueryPostgreSQLTest.java
View File

@ -7,9 +7,11 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.SortedMap;
import java.util.SortedSet;
import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.NumberedFilter;
import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException;
@ -96,4 +98,41 @@ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest {
}
}
@Test
public void testGetFilterValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception {
Calendar startTimeCalendar = Calendar.getInstance();
startTimeCalendar.set(Calendar.YEAR, 2020);
startTimeCalendar.set(Calendar.MONTH, Calendar.MARCH);
startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1);
startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0);
startTimeCalendar.set(Calendar.MINUTE, 0);
Calendar entTimeCalendar = Calendar.getInstance();
entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH);
entTimeCalendar.set(Calendar.DAY_OF_MONTH, 15);
entTimeCalendar.set(Calendar.HOUR_OF_DAY, 16);
entTimeCalendar.set(Calendar.MINUTE, 17);
List<Filter> filters = new ArrayList<>();
Filter filter = new Filter(UsageRecord.CONSUMER_ID, QueryTest.getRandomUser());
filters.add(filter);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MINUTELY);
SortedSet<NumberedFilter> numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD);
for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter);
}
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD);
for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter);
}
numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedServiceUsageRecord.class, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, 2);
for(NumberedFilter numberedFilter : numberedFilters) {
logger.debug("{}", numberedFilter);
}
}
}

15
src/test/java/org/gcube/accounting/analytics/persistence/postgresql/QueryTest.java
View File

@ -14,8 +14,10 @@ import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.datamodel.UsageRecord;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -59,13 +61,14 @@ public class QueryTest extends ContextTest {
Filter filter = new Filter(UsageRecord.CONSUMER_ID, getRandomUser());
filters.add(filter);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MINUTELY);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY);
Query query = new Query(AggregatedServiceUsageRecord.class);
query.setTemporalConstraint(temporalConstraint);
query.setFilters(filters);
String ret = query.getTimeSeriesQuery();
logger.debug(ret);
String currentScope = AccountingPersistenceBackendQuery.getScopeToQuery();
Set<String> contexts = new HashSet<>();
contexts.add(currentScope);
@ -73,6 +76,16 @@ public class QueryTest extends ContextTest {
ret = query.getTimeSeriesQuery();
logger.debug(ret);
query = new Query(AggregatedServiceUsageRecord.class);
query.setTemporalConstraint(temporalConstraint);
query.setFilters(filters);
query.setContexts(contexts);
query.setFieldOfRequesteValues(ServiceUsageRecord.CALLED_METHOD);
query.setOrderByField(AccountingPersistenceQuery.getDefaultOrderingProperties(AggregatedServiceUsageRecord.class));
ret = query.getNextPossibleValueQuery();
logger.debug(ret);
}
}