gCubeSystem
/
accounting-aggregator-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixes

This commit is contained in:
Luca Frosini 2021-04-08 11:30:13 +02:00
parent 12ca03a111
commit f7c89321ee
4 changed files with 152 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/org/gcube/accounting/aggregator/aggregation/Aggregator.java
View File

@ -295,8 +295,11 @@ public class Aggregator {
if(record.getResourceProperty(AggregatedServiceUsageRecord.CALLER_QUALIFIER)==null) {
record.setResourceProperty(AggregatedServiceUsageRecord.CALLER_QUALIFIER, AbstractServiceUsageRecord.UNKNOWN);
}
record.validate();
}else {
return;
}
record.validate();
}
record.setId(UUID.randomUUID().toString());

3
src/main/java/org/gcube/accounting/aggregator/aggregation/DesignID.java
View File

@ -5,12 +5,13 @@ package org.gcube.accounting.aggregator.aggregation;
*/
public enum DesignID {
accounting_storage("StorageUsageRecordAggregated","all"),
accounting_storage_status("StorageStatusRecordAggregated","all"),
StorageUsageRecord("StorageUsageRecordAggregated","all"),
ServiceUsageRecord("ServiceUsageRecordAggregated","all"),
AggregatedServiceUsageRecord("ServiceUsageRecordAggregated","all"),
JobUsageRecord("JobUsageRecordAggregated","all");
private String designName;
private String viewName;

143
src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java
View File

@ -3,11 +3,14 @@ package org.gcube.accounting.aggregator.plugin;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.gcube.accounting.aggregator.aggregation.AggregationType;
import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin.ElaborationType;
import org.gcube.accounting.aggregator.utility.Utility;
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -20,9 +23,9 @@ public class AccountingAggregatorPluginTest extends ContextTest {
public static final String ROOT_PROD = "/d4science.research-infrastructures.eu";
@Test
public void aggregate() throws Exception {
ContextTest.setContextByName(ROOT_DEV_SCOPE);
//ContextTest.setContextByName(ROOT_PROD);
public void aggregateJob() throws Exception {
//ContextTest.setContextByName(ROOT_DEV_SCOPE);
ContextTest.setContextByName(ROOT_PROD);
Map<String, Object> inputs = new HashMap<String, Object>();
@ -34,8 +37,120 @@ public class AccountingAggregatorPluginTest extends ContextTest {
inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name());
inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(8, 0));
inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(20, 30));
inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10));
inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59));
inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, JobUsageRecord.class.newInstance().getRecordType());
inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false);
inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true);
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1);
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1);
/*
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
*/
AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin();
logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs);
while(aggregationStartCalendar.before(aggregationEndCalendar)) {
plugin.launch(inputs);
Thread.sleep(TimeUnit.SECONDS.toMillis(20));
aggregationStartCalendar.add(aggregationType.getCalendarField(), 1);
aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
}
}
@Test
public void aggregateStorage() throws Exception {
//ContextTest.setContextByName(ROOT_DEV_SCOPE);
ContextTest.setContextByName(ROOT_PROD);
Map<String, Object> inputs = new HashMap<String, Object>();
AggregationType aggregationType = AggregationType.MONTHLY;
//type aggregation51170
inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name());
inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name());
inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10));
inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59));
inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, StorageUsageRecord.class.newInstance().getRecordType());
inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false);
inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true);
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.JULY, 1);
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1);
/*
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
*/
AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin();
logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs);
while(aggregationStartCalendar.before(aggregationEndCalendar)) {
plugin.launch(inputs);
//Thread.sleep(TimeUnit.MINUTES.toMillis(1));
aggregationStartCalendar.add(aggregationType.getCalendarField(), 1);
aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
}
}
@Test
public void aggregateService() throws Exception {
//ContextTest.setContextByName(ROOT_DEV_SCOPE);
ContextTest.setContextByName(ROOT_PROD);
Map<String, Object> inputs = new HashMap<String, Object>();
AggregationType aggregationType = AggregationType.MONTHLY;
//type aggregation
inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name());
inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name());
inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10));
inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59));
inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, ServiceUsageRecord.class.newInstance().getRecordType());
@ -43,16 +158,16 @@ public class AccountingAggregatorPluginTest extends ContextTest {
inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false);
inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true);
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, false);
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, false);
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2019, Calendar.DECEMBER, 1);
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.APRIL, 1);
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1);
/*
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
@ -94,10 +209,10 @@ public class AccountingAggregatorPluginTest extends ContextTest {
inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.RECOVERY.name());
inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, AggregationType.MONTHLY.name());
inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(8, 0));
inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(20, 30));
inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 0));
inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(24, 00));
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.FEBRUARY, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.MAY, 1);
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
@ -107,16 +222,16 @@ public class AccountingAggregatorPluginTest extends ContextTest {
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1);
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 1);
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin();
logger.debug("Going to launch {} with inputs {}",plugin.getName(), inputs);
/*
while(aggregationStartCalendar.before(aggregationEndCalendar)) {
plugin.launch(inputs);

22
src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java
View File

@ -1,6 +1,8 @@
package org.gcube.accounting.aggregator.plugin;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
@ -261,13 +263,23 @@ public class MyTest {
}
@Test
public void testUTCStartAndTime() throws ParseException{
String persistTimeString = Utility.getPersistTimeParameter(8, 00);
Date endTime = Utility.getPersistTimeDate(persistTimeString);
public void testStartAndEndTime() throws ParseException{
String pattern = new String("yyyy/MM/dd HH:mm Z");
Calendar now = Calendar.getInstance();
Calendar now = Utility.getUTCCalendarInstance();
logger.info("It's {}", Utility.getUTCDateFormat(pattern).format(now.getTime()));
String startTimeString = Utility.getPersistTimeParameter(0, 10);
Date startTime = Utility.getPersistTimeDate(startTimeString);
logger.info("Start Time {}", Utility.getUTCDateFormat(pattern).format(startTime));
String endTimeString = Utility.getPersistTimeParameter(23, 59);
Date endTime = Utility.getPersistTimeDate(endTimeString);
logger.info("End Time {}", Utility.getUTCDateFormat(pattern).format(endTime));
logger.info("{}", now.getTime().after(startTime));
logger.info("{}", now.getTime().before(endTime));
Utility.isTimeElapsed(now, endTime);
}
}