package org.gcube.accounting.aggregator.plugin; import java.util.Calendar; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import org.gcube.accounting.aggregator.ContextTest; import org.gcube.accounting.aggregator.aggregation.AggregationType; import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin.ElaborationType; import org.gcube.accounting.aggregator.utility.Utility; import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.StorageStatusRecord; import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord; import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class AccountingAggregatorPluginTest extends ContextTest { private static Logger logger = LoggerFactory.getLogger(AccountingAggregatorPluginTest.class); public static final String ROOT_PROD = "/d4science.research-infrastructures.eu"; private void aggregate(String recordType, AggregationType aggregationType, Calendar aggregationStartCalendar, Calendar aggregationEndCalendar, boolean forceRerun, boolean forceEarlyAggregation) throws Exception { Map inputs = new HashMap(); inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name()); inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name()); inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, recordType); inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, aggregationStartCalendar == null); inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, forceEarlyAggregation); inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, forceRerun); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, false); if (aggregationStartCalendar != null) { String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT .format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); } if (aggregationStartCalendar != null && aggregationEndCalendar != null) { String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT .format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate); } AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin(); logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs); if (aggregationStartCalendar != null && aggregationEndCalendar != null) { while (aggregationStartCalendar.before(aggregationEndCalendar)) { plugin.launch(inputs); Thread.sleep(TimeUnit.SECONDS.toMillis(5)); plugin = new AccountingAggregatorPlugin(); aggregationStartCalendar.add(aggregationType.getCalendarField(), 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT .format(aggregationStartCalendar.getTime()); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); } } else { plugin.launch(inputs); } } @JsonIgnore @Test public void aggregateAllServiceUsageRecord() throws Exception { String recordType = ServiceUsageRecord.class.newInstance().getRecordType(); // aggregateOneShot(recordType); aggregateEverything(recordType); } // @JsonIgnore @Test public void aggregateDailyServiceUsageRecord() throws Exception { String recordType = ServiceUsageRecord.class.newInstance().getRecordType(); boolean forceRestart = true; boolean forceEarlyAggregation = true; AggregationType aggregationType = AggregationType.DAILY; Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 6); Calendar end = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 14); while (aggregationStartCalendar.before(end)) { Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); } } // @JsonIgnore @Test public void aggregateMonthlyFirstHalf2021ServiceUsageRecord() throws Exception { String recordType = ServiceUsageRecord.class.newInstance().getRecordType(); boolean forceRestart = true; boolean forceEarlyAggregation = true; AggregationType aggregationType = AggregationType.MONTHLY; Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.FEBRUARY, 1); Calendar end = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1); while (aggregationStartCalendar.before(end)) { Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); } } // @JsonIgnore @Test public void aggregateMonthlySecondHalf2021ServiceUsageRecord() throws Exception { String recordType = ServiceUsageRecord.class.newInstance().getRecordType(); boolean forceRestart = true; boolean forceEarlyAggregation = true; AggregationType aggregationType = AggregationType.MONTHLY; Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.AUGUST, 1); Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1); while (aggregationStartCalendar.before(end)) { Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); } } @JsonIgnore @Test public void aggregateAllStorageStatusRecord() throws Exception { String recordType = StorageStatusRecord.class.newInstance().getRecordType(); // aggregateOneShot(recordType); // aggregateEverything(recordType); } // @JsonIgnore @Test public void aggregateMonthlyStorageStatusRecord() throws Exception { String recordType = StorageStatusRecord.class.newInstance().getRecordType(); boolean forceRestart = true; boolean forceEarlyAggregation = true; AggregationType aggregationType = AggregationType.MONTHLY; Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1); Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1); while (aggregationStartCalendar.before(end)) { Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); } } // @JsonIgnore @Test public void aggregateSecondHalf2022StorageStatusRecord() throws Exception { String recordType = StorageStatusRecord.class.newInstance().getRecordType(); boolean forceRestart = true; boolean forceEarlyAggregation = true; AggregationType aggregationType = AggregationType.MONTHLY; Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.JUNE, 1); Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1); while (aggregationStartCalendar.before(end)) { Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); } } @JsonIgnore @Test public void aggregateAllJobUsageRecord() throws Exception { String recordType = JobUsageRecord.class.newInstance().getRecordType(); aggregateOneShot(recordType); // aggregateEverything(recordType); } @JsonIgnore @Test public void aggregateAllStorageUsageRecord() throws Exception { String recordType = StorageUsageRecord.class.newInstance().getRecordType(); aggregateOneShot(recordType); // aggregateEverything(recordType); } public void aggregateOneShot(String recordType) throws Exception { boolean forceRestart = true; boolean forceEarlyAggregation = true; AggregationType aggregationType = AggregationType.DAILY; Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 14); Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); } public void aggregateEverything(String recordType) throws Exception { boolean forceRestart = true; boolean forceEarlyAggregation = false; AggregationType aggregationType = AggregationType.YEARLY; Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.JANUARY, 1); Calendar end = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); while (aggregationStartCalendar.before(end)) { Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); } aggregationType = AggregationType.MONTHLY; aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(end.getTimeInMillis()); end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1); while (aggregationStartCalendar.before(end)) { Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation); aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); } aggregationType = AggregationType.DAILY; aggregationStartCalendar = end; end = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 14); while (aggregationStartCalendar.before(end)) { Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1); aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, true); aggregationStartCalendar = Calendar.getInstance(); aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis()); } } @JsonIgnore @Test public void testRecovery() throws Exception { ContextTest.setContextByName(GCUBE); // ContextTest.setContextByName(ROOT_PROD); Map inputs = new HashMap(); AggregationType aggregationType = AggregationType.DAILY; inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.RECOVERY.name()); inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name()); // Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.MAY, 1); // String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); // logger.trace("{} : {}",AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, false); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); // Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 1); // String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); // logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); // inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate); AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin(); logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs); plugin.launch(inputs); // while(aggregationStartCalendar.before(aggregationEndCalendar)) { // plugin.launch(inputs); aggregationStartCalendar.add(Calendar.MONTH, 1); // aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); // inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // } } }