diff --git a/src/main/java/org/gcube/accounting/aggregator/aggregation/Aggregator.java b/src/main/java/org/gcube/accounting/aggregator/aggregation/Aggregator.java index 6cf7b05..6230635 100644 --- a/src/main/java/org/gcube/accounting/aggregator/aggregation/Aggregator.java +++ b/src/main/java/org/gcube/accounting/aggregator/aggregation/Aggregator.java @@ -295,8 +295,11 @@ public class Aggregator { if(record.getResourceProperty(AggregatedServiceUsageRecord.CALLER_QUALIFIER)==null) { record.setResourceProperty(AggregatedServiceUsageRecord.CALLER_QUALIFIER, AbstractServiceUsageRecord.UNKNOWN); } + record.validate(); + }else { + return; } - record.validate(); + } record.setId(UUID.randomUUID().toString()); diff --git a/src/main/java/org/gcube/accounting/aggregator/aggregation/DesignID.java b/src/main/java/org/gcube/accounting/aggregator/aggregation/DesignID.java index 27a700f..e1e847f 100644 --- a/src/main/java/org/gcube/accounting/aggregator/aggregation/DesignID.java +++ b/src/main/java/org/gcube/accounting/aggregator/aggregation/DesignID.java @@ -5,12 +5,13 @@ package org.gcube.accounting.aggregator.aggregation; */ public enum DesignID { - accounting_storage("StorageUsageRecordAggregated","all"), + accounting_storage_status("StorageStatusRecordAggregated","all"), StorageUsageRecord("StorageUsageRecordAggregated","all"), ServiceUsageRecord("ServiceUsageRecordAggregated","all"), AggregatedServiceUsageRecord("ServiceUsageRecordAggregated","all"), JobUsageRecord("JobUsageRecordAggregated","all"); + private String designName; private String viewName; diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java index d79b4f0..1c9f665 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java @@ -3,11 +3,14 @@ package org.gcube.accounting.aggregator.plugin; import java.util.Calendar; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.gcube.accounting.aggregator.aggregation.AggregationType; import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin.ElaborationType; import org.gcube.accounting.aggregator.utility.Utility; +import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,9 +23,9 @@ public class AccountingAggregatorPluginTest extends ContextTest { public static final String ROOT_PROD = "/d4science.research-infrastructures.eu"; @Test - public void aggregate() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); - //ContextTest.setContextByName(ROOT_PROD); + public void aggregateJob() throws Exception { + //ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(ROOT_PROD); Map inputs = new HashMap(); @@ -34,8 +37,120 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name()); - inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(8, 0)); - inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(20, 30)); + inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10)); + inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59)); + + + inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, JobUsageRecord.class.newInstance().getRecordType()); + + inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false); + + inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true); + inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); + inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); + + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1); + String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); + logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + + // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); + /* + String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); + logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate); + */ + + AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin(); + logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs); + + + + while(aggregationStartCalendar.before(aggregationEndCalendar)) { + plugin.launch(inputs); + Thread.sleep(TimeUnit.SECONDS.toMillis(20)); + aggregationStartCalendar.add(aggregationType.getCalendarField(), 1); + aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + } + + } + + + @Test + public void aggregateStorage() throws Exception { + //ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(ROOT_PROD); + + Map inputs = new HashMap(); + + + AggregationType aggregationType = AggregationType.MONTHLY; + //type aggregation51170 + inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name()); + + inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name()); + + + inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10)); + inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59)); + + + inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, StorageUsageRecord.class.newInstance().getRecordType()); + + inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false); + + inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true); + inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); + inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); + + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.JULY, 1); + String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); + logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + + // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1); + /* + String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); + logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate); + */ + + AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin(); + logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs); + + + + while(aggregationStartCalendar.before(aggregationEndCalendar)) { + plugin.launch(inputs); + //Thread.sleep(TimeUnit.MINUTES.toMillis(1)); + aggregationStartCalendar.add(aggregationType.getCalendarField(), 1); + aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + } + + } + + + @Test + public void aggregateService() throws Exception { + //ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(ROOT_PROD); + + Map inputs = new HashMap(); + + + AggregationType aggregationType = AggregationType.MONTHLY; + //type aggregation + inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name()); + + inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name()); + + + inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10)); + inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59)); inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, ServiceUsageRecord.class.newInstance().getRecordType()); @@ -43,16 +158,16 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false); inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true); - inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, false); - inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, false); + inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); + inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2019, Calendar.DECEMBER, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.APRIL, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); @@ -94,10 +209,10 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.RECOVERY.name()); inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, AggregationType.MONTHLY.name()); - inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(8, 0)); - inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(20, 30)); + inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 0)); + inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(24, 00)); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.FEBRUARY, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.MAY, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); @@ -107,16 +222,16 @@ public class AccountingAggregatorPluginTest extends ContextTest { // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 1); String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate); + AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin(); logger.debug("Going to launch {} with inputs {}",plugin.getName(), inputs); - /* while(aggregationStartCalendar.before(aggregationEndCalendar)) { plugin.launch(inputs); diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java index 44accf7..861610f 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java @@ -1,6 +1,8 @@ package org.gcube.accounting.aggregator.plugin; +import java.text.DateFormat; import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.Map; @@ -261,13 +263,23 @@ public class MyTest { } @Test - public void testUTCStartAndTime() throws ParseException{ - String persistTimeString = Utility.getPersistTimeParameter(8, 00); - Date endTime = Utility.getPersistTimeDate(persistTimeString); + public void testStartAndEndTime() throws ParseException{ + String pattern = new String("yyyy/MM/dd HH:mm Z"); - Calendar now = Calendar.getInstance(); + Calendar now = Utility.getUTCCalendarInstance(); + logger.info("It's {}", Utility.getUTCDateFormat(pattern).format(now.getTime())); + + String startTimeString = Utility.getPersistTimeParameter(0, 10); + Date startTime = Utility.getPersistTimeDate(startTimeString); + logger.info("Start Time {}", Utility.getUTCDateFormat(pattern).format(startTime)); + + String endTimeString = Utility.getPersistTimeParameter(23, 59); + Date endTime = Utility.getPersistTimeDate(endTimeString); + logger.info("End Time {}", Utility.getUTCDateFormat(pattern).format(endTime)); + + logger.info("{}", now.getTime().after(startTime)); + logger.info("{}", now.getTime().before(endTime)); - Utility.isTimeElapsed(now, endTime); } }