diff --git a/pom.xml b/pom.xml index bc66da3..d719f53 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.gcube.vremanagement smart-executor-bom - 2.0.0-SNAPSHOT + 3.1.0-SNAPSHOT pom import @@ -63,6 +63,11 @@ accounting-analytics-persistence-postgresql [1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT) + + org.gcube.accounting + accounting-analytics + [3.0.0, 4.0.0-SNAPSHOT) + org.slf4j slf4j-api @@ -80,7 +85,6 @@ test - @@ -104,5 +108,4 @@ - \ No newline at end of file diff --git a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java index a3e4056..36e40df 100644 --- a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java +++ b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java @@ -10,6 +10,7 @@ import org.gcube.accounting.aggregator.aggregation.AggregationType; import org.gcube.accounting.aggregator.aggregation.Aggregator; import org.gcube.accounting.aggregator.directory.FileSystemDirectoryStructure; import org.gcube.accounting.aggregator.persist.Persist; +import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin; import org.gcube.accounting.aggregator.status.AggregationState; import org.gcube.accounting.aggregator.status.AggregationStatus; import org.gcube.accounting.aggregator.utility.Constant; @@ -178,15 +179,14 @@ public class Elaborator { * (even rare) where both check are valid because the first invocation happen * before midnight and the second after midnight (so in the next day). */ - // if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) { + if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) { // Persist persist = new Persist(aggregationStatus, srcBucket, dstBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); persist.recover(); - /* }else{ logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime)); } - */ + } protected File getOriginalRecordsBackupFile(File elaborationDirectory, String name) throws Exception { diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java index 994d6e2..8a7bda1 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java @@ -105,13 +105,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2018, Calendar.APRIL, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.MAY, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); @@ -161,13 +161,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.APRIL, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.APRIL, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.MAY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); @@ -217,13 +217,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JUNE, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/CouchBaseConnectorTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/CouchBaseConnectorTest.java deleted file mode 100644 index a5379e3..0000000 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/CouchBaseConnectorTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package org.gcube.accounting.aggregator.plugin; - -import java.util.Calendar; -import java.util.Date; -import java.util.List; - -import org.gcube.accounting.aggregator.aggregation.AggregationInfo; -import org.gcube.accounting.aggregator.aggregation.AggregationType; -import org.gcube.accounting.aggregator.persistence.PostgreSQLConnector; -import org.gcube.accounting.aggregator.status.AggregationState; -import org.gcube.accounting.aggregator.status.AggregationStatus; -import org.gcube.accounting.aggregator.utility.Utility; -import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; -import org.gcube.documentstore.records.DSMapper; -import org.junit.Ignore; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CouchBaseConnectorTest extends ContextTest { - - private static Logger logger = LoggerFactory.getLogger(AccountingAggregatorPluginTest.class); - - private PostgreSQLConnector postgreSQLConnector; - - public CouchBaseConnectorTest() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); - postgreSQLConnector = new PostgreSQLConnector(); - } - - protected void analyseAggregationStatus(AggregationStatus aggregationStatus) throws Exception { - if(aggregationStatus.getPrevious()!=null) { - analyseAggregationStatus(aggregationStatus.getPrevious()); - } - if(aggregationStatus.getOriginalRecordsNumber()!=0) { - postgreSQLConnector.upsertAggregationStatus(aggregationStatus); - } - - } - - @Ignore - @Test - public void insertAllInPostgreSQL() throws Exception { - ContextTest.setContextByName(ROOT_DEV_SCOPE); - List aggregationStatuses = postgreSQLConnector.getAll(); - for(AggregationStatus aggregationStatus : aggregationStatuses) { - analyseAggregationStatus(aggregationStatus); - } - } - - @Test - public void getLastTest() throws Exception { - AggregationStatus aggregationStatus = postgreSQLConnector.getLast(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, null, null); - logger.debug("Last : {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - } - - @Test - public void getUnterminatedTest() throws Exception{ - List aggregationStatuses = postgreSQLConnector.getUnterminated(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, null, null); - for(AggregationStatus aggregationStatus : aggregationStatuses){ - logger.debug("Unterminated : {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - } - } - - @Test - public void getLastTestWithConstraint() throws Exception { - Calendar aggregationStart = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1); - Calendar aggregationEnd = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 31); - - AggregationStatus aggregationStatus = postgreSQLConnector.getLast(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, aggregationStart.getTime(), aggregationEnd.getTime()); - logger.info("Last : {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - } - - @Test - public void getUnterminatedTestWithConstraint() throws Exception{ - Calendar aggregationStart = Utility.getAggregationStartCalendar(2017, Calendar.APRIL, 1); - Calendar aggregationEnd = Utility.getAggregationStartCalendar(2017, Calendar.APRIL, 30); - - List aggregationStatuses = postgreSQLConnector.getUnterminated(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, aggregationStart.getTime(), aggregationEnd.getTime()); - for(AggregationStatus aggregationStatus : aggregationStatuses){ - logger.info("Unterminated : {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - } - } - - @Test - public void getAggregationStatusTest() throws Exception{ - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 15); - AggregationStatus aggregationStatus = postgreSQLConnector.getAggregationStatus(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, aggregationStartCalendar.getTime()); - logger.debug("{}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - } - - @Test - public void aggregationStatusTest() throws Exception { - int toRemove = -36; - - Calendar today = Utility.getUTCCalendarInstance(); - today.add(Calendar.DAY_OF_YEAR, toRemove); - - String aggregationStartDateString = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(today.getTime()); - Date aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.parse(aggregationStartDateString); - - Calendar tomorrow = Utility.getUTCCalendarInstance(); - tomorrow.add(Calendar.DAY_OF_YEAR, toRemove+1); - String aggregationEndDateString = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(tomorrow.getTime()); - Date aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.parse(aggregationEndDateString); - - - AggregationInfo aggregation = new AggregationInfo(ServiceUsageRecord.class.newInstance().getRecordType(), AggregationType.DAILY, aggregationStartDate, aggregationEndDate); - String aggregationString = DSMapper.getObjectMapper().writeValueAsString(aggregation); - logger.debug("{} : {}", AggregationInfo.class.getSimpleName(), aggregationString); - - AggregationStatus aggregationStatus = new AggregationStatus(aggregation); - aggregationStatus.setContext("TEST_CONTEXT"); - - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - // Set to true just for one test and restore to false - boolean sync = true; - - Calendar startedStart = Utility.getUTCCalendarInstance(); - aggregationStatus.setAggregationState(AggregationState.STARTED, startedStart, sync); - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - aggregationStatus.setRecordNumbers(100, 72, 0); - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - Calendar aggregatedStart = Utility.getUTCCalendarInstance(); - aggregationStatus.setAggregationState(AggregationState.AGGREGATED, aggregatedStart, sync); - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - Calendar addedStart = Utility.getUTCCalendarInstance(); - aggregationStatus.setAggregationState(AggregationState.ADDED, addedStart, sync); - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - Calendar deletedStart = Utility.getUTCCalendarInstance(); - aggregationStatus.setAggregationState(AggregationState.DELETED, deletedStart, sync); - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - Calendar completedStart = Utility.getUTCCalendarInstance(); - aggregationStatus.setAggregationState(AggregationState.COMPLETED, completedStart, sync); - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - } - - @Test - public void createStartedElaboration() throws Exception { - - Calendar start = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 15); - String aggregationStartDateString = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(start.getTime()); - Date aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.parse(aggregationStartDateString); - - Calendar end = Utility.getUTCCalendarInstance(); - end.setTime(aggregationStartDate); - end.add(Calendar.DAY_OF_MONTH, 1); - String aggregationEndDateString = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(end.getTime()); - Date aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.parse(aggregationEndDateString); - - - AggregationInfo aggregation = new AggregationInfo(ServiceUsageRecord.class.newInstance().getRecordType(), AggregationType.DAILY, aggregationStartDate, aggregationEndDate); - String aggregationString = DSMapper.getObjectMapper().writeValueAsString(aggregation); - logger.debug("{} : {}", AggregationInfo.class.getSimpleName(), aggregationString); - - AggregationStatus aggregationStatus = new AggregationStatus(aggregation); - aggregationStatus.setContext("TEST_CONTEXT"); - - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - // Set to true just for one test and restore to false - boolean sync = true; - - Calendar startedStart = Utility.getUTCCalendarInstance(); - aggregationStatus.setAggregationState(AggregationState.STARTED, startedStart, sync); - logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - - } -} diff --git a/src/test/resources/META-INF/plugin.properties b/src/test/resources/META-INF/AccountingAggregatorPlugin.properties similarity index 100% rename from src/test/resources/META-INF/plugin.properties rename to src/test/resources/META-INF/AccountingAggregatorPlugin.properties