/** * */ package org.gcube.accounting.analytics.persistence.postgresql; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.NumberedFilter; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.exception.DuplicatedKeyFilterException; import org.gcube.accounting.analytics.exception.KeyException; import org.gcube.accounting.analytics.exception.ValueException; import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryConfiguration; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory; import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedJobUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageStatusRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; import org.gcube.accounting.datamodel.basetypes.AbstractStorageUsageRecord.DataType; import org.gcube.accounting.utility.postgresql.RecordToDBMapping; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.Record; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author Luca Frosini (ISTI - CNR) */ public class AccountingPersistenceQueryPostgreSQLTest extends ContextTest { private static final Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryPostgreSQLTest.class); protected AccountingPersistenceQueryPostgreSQL accountingPersistenceQueryPostgreSQL; @Before public void before() throws Exception { AccountingPersistenceBackendQueryConfiguration configuration = new AccountingPersistenceBackendQueryConfiguration( AccountingPersistenceQueryPostgreSQL.class); accountingPersistenceQueryPostgreSQL = new AccountingPersistenceQueryPostgreSQL(); accountingPersistenceQueryPostgreSQL.prepareConnection(configuration); } protected List getFilters(){ List filters = new ArrayList<>(); String user = QueryTest.getRandomUser(); logger.debug("Going to query filtering with user {}", user); Filter filter = new Filter(UsageRecord.CONSUMER_ID, user); filters.add(filter); return filters; } protected TemporalConstraint getTemporalConstraint() { Calendar startTimeCalendar = Calendar.getInstance(); startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); startTimeCalendar.set(Calendar.YEAR, 2015); startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); startTimeCalendar.set(Calendar.MINUTE, 0); Calendar entTimeCalendar = Calendar.getInstance(); entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); entTimeCalendar.set(Calendar.YEAR, 2021); entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH); entTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); entTimeCalendar.set(Calendar.MINUTE, 0); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.MONTHLY); return temporalConstraint; } protected List>> getClassesToTest(){ List>> classes = new ArrayList>>(); classes.add(AggregatedServiceUsageRecord.class); classes.add(AggregatedStorageUsageRecord.class); classes.add(AggregatedJobUsageRecord.class); return classes; } @Test public void testGetNoContextTimeSeriesYearly() throws Exception { List filters = getFilters(); AggregationMode aggregationMode = AggregationMode.YEARLY; Calendar startTimeCalendar = Calendar.getInstance(); startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); startTimeCalendar.set(Calendar.YEAR, 2015); startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); startTimeCalendar.set(Calendar.MINUTE, 0); Calendar entTimeCalendar = Calendar.getInstance(); entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); entTimeCalendar.set(Calendar.YEAR, 2021); entTimeCalendar.set(Calendar.MONTH, Calendar.DECEMBER); entTimeCalendar.set(Calendar.DAY_OF_MONTH, 31); entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); entTimeCalendar.set(Calendar.MINUTE, 0); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode); Class> clz = AggregatedServiceUsageRecord.class; logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name()); temporalConstraint.setAggregationMode(aggregationMode); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } @Test public void testGetNoContextTimeSeriesMonthly() throws Exception { List filters = getFilters(); AggregationMode aggregationMode = AggregationMode.MONTHLY; Calendar startTimeCalendar = Calendar.getInstance(); startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); startTimeCalendar.set(Calendar.YEAR, 2015); startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); startTimeCalendar.set(Calendar.MINUTE, 0); Calendar entTimeCalendar = Calendar.getInstance(); entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); entTimeCalendar.set(Calendar.YEAR, 2021); entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER); entTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); entTimeCalendar.set(Calendar.MINUTE, 0); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode); Class> clz = AggregatedServiceUsageRecord.class; logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name()); temporalConstraint.setAggregationMode(aggregationMode); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } @Test public void testGetNoContextTimeSeriesDaily() throws Exception { List filters = getFilters(); AggregationMode aggregationMode = AggregationMode.DAILY; Calendar startTimeCalendar = Calendar.getInstance(); startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); startTimeCalendar.set(Calendar.YEAR, 2021); startTimeCalendar.set(Calendar.MONTH, Calendar.OCTOBER); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26); startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); startTimeCalendar.set(Calendar.MINUTE, 0); Calendar entTimeCalendar = Calendar.getInstance(); entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); entTimeCalendar.set(Calendar.YEAR, 2021); entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER); entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26); entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); entTimeCalendar.set(Calendar.MINUTE, 0); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode); Class> clz = AggregatedServiceUsageRecord.class; logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name()); temporalConstraint.setAggregationMode(aggregationMode); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } @Test public void testGetNoContextTimeSeriesHourly() throws Exception { List filters = getFilters(); AggregationMode aggregationMode = AggregationMode.HOURLY; Calendar startTimeCalendar = Calendar.getInstance(); startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); startTimeCalendar.set(Calendar.YEAR, 2021); startTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26); startTimeCalendar.set(Calendar.HOUR_OF_DAY, 00); startTimeCalendar.set(Calendar.MINUTE, 0); Calendar entTimeCalendar = Calendar.getInstance(); entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); entTimeCalendar.set(Calendar.YEAR, 2021); entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER); entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26); entTimeCalendar.set(Calendar.HOUR_OF_DAY, 23); entTimeCalendar.set(Calendar.MINUTE, 0); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode); Class> clz = AggregatedServiceUsageRecord.class; logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name()); temporalConstraint.setAggregationMode(aggregationMode); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } @Test public void testGetNoContextTimeSeriesMinutely() throws Exception { List filters = getFilters(); AggregationMode aggregationMode = AggregationMode.MINUTELY; Calendar startTimeCalendar = Calendar.getInstance(); startTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); startTimeCalendar.set(Calendar.YEAR, 2021); startTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 26); startTimeCalendar.set(Calendar.HOUR_OF_DAY, 17); startTimeCalendar.set(Calendar.MINUTE, 15); Calendar entTimeCalendar = Calendar.getInstance(); entTimeCalendar.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); entTimeCalendar.set(Calendar.YEAR, 2021); entTimeCalendar.set(Calendar.MONTH, Calendar.NOVEMBER); entTimeCalendar.set(Calendar.DAY_OF_MONTH, 26); entTimeCalendar.set(Calendar.HOUR_OF_DAY, 17); entTimeCalendar.set(Calendar.MINUTE, 15); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), aggregationMode); Class> clz = AggregatedServiceUsageRecord.class; logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {} - Aggregation Mode {}\n", RecordToDBMapping.getRecordTypeByClass(clz), aggregationMode.name()); temporalConstraint.setAggregationMode(aggregationMode); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } @Test public void testGetNoContextTimeSeries() throws Exception { List filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); List>> classes = getClassesToTest(); for(Class> clz : classes) { logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } } @Test public void testTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { List filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); List>> classes = getClassesToTest(); for(Class> clz : classes) { logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap timeseries = accountingPersistenceQueryPostgreSQL.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } } @Test public void testContextTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { List filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); List>> classes = getClassesToTest(); Set contexts = new HashSet<>(); contexts.add("/gcube/devsec/devVRE"); contexts.add("/gcube/devNext/NextNext"); for(Class> clz : classes) { logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setContexts(contexts); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap> contextTimeseries = accountingPersistenceQueryPostgreSQL.getContextTimeSeries(); for(Filter f : contextTimeseries.keySet()) { logger.debug("{}", f); SortedMap timeseries = contextTimeseries.get(f); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } } } @Test public void testGetFilterValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { List filters = getFilters(); TemporalConstraint temporalConstraint = getTemporalConstraint(); List>> classes = getClassesToTest(); Map>, String> keysToRequest = new HashMap<>(); keysToRequest.put(AggregatedServiceUsageRecord.class, AggregatedServiceUsageRecord.CALLED_METHOD); keysToRequest.put(AggregatedStorageUsageRecord.class, AggregatedStorageUsageRecord.CONSUMER_ID); keysToRequest.put(AggregatedJobUsageRecord.class, AggregatedJobUsageRecord.JOB_NAME); for(Class> clz : classes) { logger.debug("----------------------------------------------------------------------"); logger.debug("Going to query {}\n", RecordToDBMapping.getRecordTypeByClass(clz)); String key = keysToRequest.get(clz); accountingPersistenceQueryPostgreSQL.setRequestedRecords(clz); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); SortedSet numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key); for(NumberedFilter numberedFilter : numberedFilters) { logger.debug("{}", numberedFilter); } accountingPersistenceQueryPostgreSQL.setFilters(filters); numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key); for(NumberedFilter numberedFilter : numberedFilters) { logger.debug("{}", numberedFilter); } accountingPersistenceQueryPostgreSQL.setFilters(null); logger.debug("----Only first 3 results"); numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3); for(NumberedFilter numberedFilter : numberedFilters) { logger.debug("{}", numberedFilter); } accountingPersistenceQueryPostgreSQL.setFilters(filters); logger.debug("----Only first 3 results"); numberedFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(key, 3); for(NumberedFilter numberedFilter : numberedFilters) { logger.debug("{}", numberedFilter); } } } @Test public void testTopValues() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { String orderingProperty = AccountingPersistenceQuery.getDefaultOrderingProperties(AggregatedServiceUsageRecord.class); List filters = new ArrayList<>(); Filter filter = new Filter(UsageRecord.CONSUMER_ID, QueryTest.getRandomUser()); filters.add(filter); TemporalConstraint temporalConstraint = getTemporalConstraint(); accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedServiceUsageRecord.class); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); accountingPersistenceQueryPostgreSQL.setFilters(filters); SortedMap> topTimeSeries = accountingPersistenceQueryPostgreSQL.getTopValues(AggregatedServiceUsageRecord.CALLED_METHOD, orderingProperty, null); for(NumberedFilter numberedFilter : topTimeSeries.keySet()) { logger.debug("{}", numberedFilter); SortedMap timeseries = topTimeSeries.get(numberedFilter); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } } @Test public void testGetRecord() throws Exception { String type = RecordToDBMapping.getRecordTypeByClass(AggregatedServiceUsageRecord.class); Record record = accountingPersistenceQueryPostgreSQL.getRecord("7c3fcb31-6909-451d-bae0-633b3bec0c21", type); logger.debug("{}", record); } @Test public void storageStatusTest() throws Exception { accountingPersistenceQueryPostgreSQL.setRequestedRecords(AggregatedStorageStatusRecord.class); Collection myFilters = accountingPersistenceQueryPostgreSQL.getFilterValues(AggregatedStorageStatusRecord.DATA_TYPE); accountingPersistenceQueryPostgreSQL.setFilters(myFilters); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(getTemporalConstraint()); accountingPersistenceQueryPostgreSQL.getTimeSeries(); } @Test public void testGetSpaceTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { TemporalConstraint temporalConstraint = getTemporalConstraint(); accountingPersistenceQueryPostgreSQL.setTemporalConstraint(temporalConstraint); Set dataTypes = new HashSet<>(); dataTypes.add(DataType.STORAGE.name()); SortedMap> getSpaceTimeSeries = accountingPersistenceQueryPostgreSQL.getSpaceTimeSeries(dataTypes); for(Filter filter : getSpaceTimeSeries.keySet()) { logger.debug("{}", filter); SortedMap timeseries = getSpaceTimeSeries.get(filter); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } } @Test public void testStorageTimeSeries() throws DuplicatedKeyFilterException, KeyException, ValueException, Exception { AccountingPersistenceQuery apq = AccountingPersistenceQueryFactory.getInstance(); Calendar startTimeCalendar = Calendar.getInstance(); startTimeCalendar.set(Calendar.YEAR, 2017); startTimeCalendar.set(Calendar.MONTH, Calendar.JANUARY); startTimeCalendar.set(Calendar.DAY_OF_MONTH, 1); startTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); startTimeCalendar.set(Calendar.MINUTE, 0); Calendar entTimeCalendar = Calendar.getInstance(); entTimeCalendar.set(Calendar.YEAR, 2021); entTimeCalendar.set(Calendar.MONTH, Calendar.MARCH); entTimeCalendar.set(Calendar.DAY_OF_MONTH, 22); entTimeCalendar.set(Calendar.HOUR_OF_DAY, 0); entTimeCalendar.set(Calendar.MINUTE, 0); TemporalConstraint temporalConstraint = new TemporalConstraint(startTimeCalendar.getTimeInMillis(), entTimeCalendar.getTimeInMillis(), AggregationMode.DAILY); apq.setRequestedRecords(AggregatedStorageStatusRecord.class); apq.setTemporalConstraint(temporalConstraint); SortedMap timeseries = apq.getTimeSeries(); for(Calendar c : timeseries.keySet()) { Info info = timeseries.get(c); logger.debug("{}", info); } } }