318 lines
15 KiB
Java
318 lines
15 KiB
Java
package org.gcube.accounting.aggregator.plugin;
|
|
|
|
import java.util.Calendar;
|
|
import java.util.HashMap;
|
|
import java.util.Map;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import org.gcube.accounting.aggregator.ContextTest;
|
|
import org.gcube.accounting.aggregator.aggregation.AggregationType;
|
|
import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin.ElaborationType;
|
|
import org.gcube.accounting.aggregator.utility.Utility;
|
|
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
|
|
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
|
import org.gcube.accounting.datamodel.usagerecords.StorageStatusRecord;
|
|
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
|
|
import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore;
|
|
import org.junit.Test;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
public class AccountingAggregatorPluginTest extends ContextTest {
|
|
|
|
private static Logger logger = LoggerFactory.getLogger(AccountingAggregatorPluginTest.class);
|
|
|
|
public static final String ROOT_PROD = "/d4science.research-infrastructures.eu";
|
|
|
|
private void aggregate(String recordType, AggregationType aggregationType, Calendar aggregationStartCalendar,
|
|
Calendar aggregationEndCalendar, boolean forceRerun, boolean forceEarlyAggregation, boolean skipAggregation) throws Exception {
|
|
Map<String, Object> inputs = new HashMap<String, Object>();
|
|
|
|
inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name());
|
|
|
|
inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name());
|
|
inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, recordType);
|
|
|
|
inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER,
|
|
aggregationStartCalendar == null);
|
|
|
|
inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, forceEarlyAggregation);
|
|
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, forceRerun);
|
|
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, false);
|
|
|
|
inputs.put(AccountingAggregatorPlugin.SKIP_AGGREGATION, skipAggregation);
|
|
|
|
if (aggregationStartCalendar != null) {
|
|
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT
|
|
.format(aggregationStartCalendar.getTime());
|
|
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER,
|
|
aggregationStartDate);
|
|
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
|
}
|
|
|
|
if (aggregationStartCalendar != null && aggregationEndCalendar != null) {
|
|
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT
|
|
.format(aggregationEndCalendar.getTime());
|
|
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER,
|
|
aggregationEndDate);
|
|
inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
|
|
}
|
|
|
|
AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin();
|
|
logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs);
|
|
|
|
if (aggregationStartCalendar != null && aggregationEndCalendar != null) {
|
|
while (aggregationStartCalendar.before(aggregationEndCalendar)) {
|
|
plugin.launch(inputs);
|
|
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
|
|
plugin = new AccountingAggregatorPlugin();
|
|
aggregationStartCalendar.add(aggregationType.getCalendarField(), 1);
|
|
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT
|
|
.format(aggregationStartCalendar.getTime());
|
|
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
|
}
|
|
} else {
|
|
plugin.launch(inputs);
|
|
}
|
|
|
|
}
|
|
|
|
@JsonIgnore
|
|
@Test
|
|
public void aggregateAllServiceUsageRecord() throws Exception {
|
|
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
|
// aggregateOneShot(recordType);
|
|
aggregateEverything(recordType);
|
|
}
|
|
|
|
|
|
// @JsonIgnore
|
|
@Test
|
|
public void aggregateMonthlyFirstHalf2022ServiceUsageRecord() throws Exception {
|
|
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
|
|
|
boolean forceRestart = true;
|
|
boolean forceEarlyAggregation = true;
|
|
|
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.FEBRUARY, 1);
|
|
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.JULY, 1);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
}
|
|
|
|
// @JsonIgnore
|
|
@Test
|
|
public void aggregateMonthlySecondHalf2022ServiceUsageRecord() throws Exception {
|
|
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
|
|
|
boolean forceRestart = true;
|
|
boolean forceEarlyAggregation = true;
|
|
|
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.JULY, 1);
|
|
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
}
|
|
|
|
@JsonIgnore
|
|
@Test
|
|
public void aggregateMonthlyFirstHalf2023ServiceUsageRecord() throws Exception {
|
|
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
|
|
|
boolean forceRestart = true;
|
|
boolean forceEarlyAggregation = true;
|
|
|
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
|
|
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JULY, 1);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
}
|
|
|
|
@JsonIgnore
|
|
@Test
|
|
public void aggregateMonthlySecondHalf2023ServiceUsageRecord() throws Exception {
|
|
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
|
|
|
boolean forceRestart = true;
|
|
boolean forceEarlyAggregation = true;
|
|
|
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.JULY, 1);
|
|
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
}
|
|
|
|
|
|
// @JsonIgnore
|
|
@Test
|
|
public void aggregateMonthlyStorageStatusRecord() throws Exception {
|
|
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
|
|
boolean forceRestart = true;
|
|
boolean forceEarlyAggregation = true;
|
|
|
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.NOVEMBER, 1);
|
|
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.JUNE, 1);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
}
|
|
|
|
// @JsonIgnore
|
|
@Test
|
|
public void aggregateSecondHalf2022StorageStatusRecord() throws Exception {
|
|
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
|
|
|
|
boolean forceRestart = true;
|
|
boolean forceEarlyAggregation = true;
|
|
|
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.SEPTEMBER, 1);
|
|
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
}
|
|
|
|
@JsonIgnore
|
|
@Test
|
|
public void aggregateAllStorageStatusRecord() throws Exception {
|
|
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
|
|
// aggregateOneShot(recordType);
|
|
// aggregateEverything(recordType);
|
|
}
|
|
|
|
// @JsonIgnore
|
|
@Test
|
|
public void aggregateAllJobUsageRecord() throws Exception {
|
|
String recordType = JobUsageRecord.class.newInstance().getRecordType();
|
|
aggregateOneShot(recordType);
|
|
// aggregateEverything(recordType);
|
|
}
|
|
|
|
// @JsonIgnore
|
|
@Test
|
|
public void aggregateAllStorageUsageRecord() throws Exception {
|
|
String recordType = StorageUsageRecord.class.newInstance().getRecordType();
|
|
aggregateOneShot(recordType);
|
|
// aggregateEverything(recordType);
|
|
}
|
|
|
|
|
|
public void aggregateOneShot(String recordType) throws Exception {
|
|
boolean forceRestart = true;
|
|
boolean forceEarlyAggregation = true;
|
|
AggregationType aggregationType = AggregationType.DAILY;
|
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 20);
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
}
|
|
|
|
public void aggregateEverything(String recordType) throws Exception {
|
|
|
|
boolean forceRestart = true;
|
|
boolean forceEarlyAggregation = false;
|
|
|
|
AggregationType aggregationType = AggregationType.YEARLY;
|
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.JANUARY, 1);
|
|
Calendar end = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
|
|
aggregationType = AggregationType.MONTHLY;
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(end.getTimeInMillis());
|
|
end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
|
|
aggregationType = AggregationType.DAILY;
|
|
aggregationStartCalendar = end;
|
|
end = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 14);
|
|
while (aggregationStartCalendar.before(end)) {
|
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, true, false);
|
|
aggregationStartCalendar = Calendar.getInstance();
|
|
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
|
|
}
|
|
|
|
}
|
|
|
|
@JsonIgnore
|
|
@Test
|
|
public void testRecovery() throws Exception {
|
|
ContextTest.setContextByName(GCUBE);
|
|
// ContextTest.setContextByName(ROOT_PROD);
|
|
|
|
Map<String, Object> inputs = new HashMap<String, Object>();
|
|
|
|
AggregationType aggregationType = AggregationType.DAILY;
|
|
|
|
inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.RECOVERY.name());
|
|
inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name());
|
|
|
|
|
|
// Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.MAY, 1);
|
|
// String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
|
// logger.trace("{} : {}",AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
|
// inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
|
|
|
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, false);
|
|
|
|
|
|
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
|
|
// Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 1);
|
|
// String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
|
|
// logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
|
|
// inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
|
|
|
|
AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin();
|
|
logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs);
|
|
|
|
plugin.launch(inputs);
|
|
|
|
|
|
// while(aggregationStartCalendar.before(aggregationEndCalendar)) {
|
|
// plugin.launch(inputs); aggregationStartCalendar.add(Calendar.MONTH, 1);
|
|
// aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
|
// inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
|
// }
|
|
|
|
}
|
|
|
|
}
|