From bdddde38e87ec0275d1d4e66bd54216d094b25ab Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Thu, 29 Jul 2021 12:35:12 +0200 Subject: [PATCH 1/7] Porting plugin to smart-executor 3 --- pom.xml | 9 ++++++--- ....properties => AccountingAggregatorPlugin.properties} | 0 2 files changed, 6 insertions(+), 3 deletions(-) rename src/test/resources/META-INF/{plugin.properties => AccountingAggregatorPlugin.properties} (100%) diff --git a/pom.xml b/pom.xml index 0f3f12f..dd1f3be 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.gcube.vremanagement smart-executor-bom - 2.0.0-SNAPSHOT + 3.0.0 pom import @@ -70,6 +70,11 @@ accounting-analytics-persistence-postgresql [1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT) + + org.gcube.accounting + accounting-analytics + [3.0.0, 4.0.0-SNAPSHOT) + org.slf4j slf4j-api @@ -87,7 +92,6 @@ test - @@ -134,5 +138,4 @@ - \ No newline at end of file diff --git a/src/test/resources/META-INF/plugin.properties b/src/test/resources/META-INF/AccountingAggregatorPlugin.properties similarity index 100% rename from src/test/resources/META-INF/plugin.properties rename to src/test/resources/META-INF/AccountingAggregatorPlugin.properties From d7677d5b513f9e058ef97b6fc469676518827318 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Thu, 29 Jul 2021 14:06:11 +0200 Subject: [PATCH 2/7] updating test --- .../plugin/AccountingAggregatorPluginTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java index 994d6e2..8a7bda1 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java @@ -105,13 +105,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2018, Calendar.APRIL, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.MAY, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); @@ -161,13 +161,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.APRIL, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.APRIL, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.MAY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); @@ -217,13 +217,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JUNE, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); From 0eb3d2fcad9dec72d790fc06623e4c0628271168 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Thu, 29 Jul 2021 14:06:48 +0200 Subject: [PATCH 3/7] removed uneeded import --- .../org/gcube/accounting/aggregator/elaboration/Elaborator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java index a42d95e..2b178af 100644 --- a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java +++ b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java @@ -12,7 +12,6 @@ import org.gcube.accounting.aggregator.directory.FileSystemDirectoryStructure; import org.gcube.accounting.aggregator.persist.Persist; import org.gcube.accounting.aggregator.persistence.CouchBaseConnector; import org.gcube.accounting.aggregator.persistence.CouchBaseConnector.SUFFIX; -import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin; import org.gcube.accounting.aggregator.status.AggregationState; import org.gcube.accounting.aggregator.status.AggregationStatus; import org.gcube.accounting.aggregator.utility.Constant; From ac30596d7066e6d09c6dec62b1e169aa54e8f004 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 2 Nov 2021 14:22:44 +0100 Subject: [PATCH 4/7] Restored check on time --- .../gcube/accounting/aggregator/elaboration/Elaborator.java | 5 ++--- .../aggregator/plugin/AccountingAggregatorPluginTest.java | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java index a42d95e..58992b4 100644 --- a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java +++ b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java @@ -184,15 +184,14 @@ public class Elaborator { * (even rare) where both check are valid because the first invocation happen * before midnight and the second after midnight (so in the next day). */ - // if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) { + if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) { // Persist persist = new Persist(aggregationStatus, srcBucket, dstBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); Persist persist = new Persist(aggregationStatus, srcBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); persist.recover(); - /* }else{ logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime)); } - */ + } protected File getOriginalRecordsBackupFile(File elaborationDirectory, String name) throws Exception { diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java index 994d6e2..ccaf68b 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java @@ -217,13 +217,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JUNE, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); From 961f563d4280c6e8b926103674999a7b94d8b2a7 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 2 Nov 2021 14:50:31 +0100 Subject: [PATCH 5/7] Switched to smart-executor-bom 3.1.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dd1f3be..2da81b5 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.gcube.vremanagement smart-executor-bom - 3.0.0 + 3.1.0-SNAPSHOT pom import From d574715763d30bacf2ba165616e2c6c685f2c1c4 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 2 Nov 2021 14:50:43 +0100 Subject: [PATCH 6/7] Added missing import --- .../org/gcube/accounting/aggregator/elaboration/Elaborator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java index 9e2b04a..58992b4 100644 --- a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java +++ b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java @@ -12,6 +12,7 @@ import org.gcube.accounting.aggregator.directory.FileSystemDirectoryStructure; import org.gcube.accounting.aggregator.persist.Persist; import org.gcube.accounting.aggregator.persistence.CouchBaseConnector; import org.gcube.accounting.aggregator.persistence.CouchBaseConnector.SUFFIX; +import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin; import org.gcube.accounting.aggregator.status.AggregationState; import org.gcube.accounting.aggregator.status.AggregationStatus; import org.gcube.accounting.aggregator.utility.Constant; From e84ec43c5a7c4e6098c3c52954b3d851de80eff1 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 2 Nov 2021 15:10:55 +0100 Subject: [PATCH 7/7] Fixed test class name --- ...uchBaseConnectorTest.java => PostgreSQLConnectorTest.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/test/java/org/gcube/accounting/aggregator/plugin/{CouchBaseConnectorTest.java => PostgreSQLConnectorTest.java} (98%) diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/CouchBaseConnectorTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/PostgreSQLConnectorTest.java similarity index 98% rename from src/test/java/org/gcube/accounting/aggregator/plugin/CouchBaseConnectorTest.java rename to src/test/java/org/gcube/accounting/aggregator/plugin/PostgreSQLConnectorTest.java index ab034ed..3d1db82 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/CouchBaseConnectorTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/PostgreSQLConnectorTest.java @@ -18,13 +18,13 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CouchBaseConnectorTest extends ContextTest { +public class PostgreSQLConnectorTest extends ContextTest { private static Logger logger = LoggerFactory.getLogger(AccountingAggregatorPluginTest.class); private PostgreSQLConnector postgreSQLConnector; - public CouchBaseConnectorTest() throws Exception { + public PostgreSQLConnectorTest() throws Exception { ContextTest.setContextByName(ROOT_DEV_SCOPE); postgreSQLConnector = new PostgreSQLConnector(); }