From 3a90441a22450da63cefd529a3f735f429a553ce Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Thu, 11 Nov 2021 14:29:09 +0100 Subject: [PATCH] Removed no more needed persist start and end time check --- .../elaboration/AggregatorManager.java | 9 ++- .../aggregator/elaboration/Elaborator.java | 15 ++--- .../elaboration/RecoveryManager.java | 13 +++-- .../persistence/PostgreSQLConnector.java | 2 +- .../plugin/AccountingAggregatorPlugin.java | 58 ++++++++----------- .../aggregator/utility/Utility.java | 22 +++---- 6 files changed, 58 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/gcube/accounting/aggregator/elaboration/AggregatorManager.java b/src/main/java/org/gcube/accounting/aggregator/elaboration/AggregatorManager.java index f74b73e..99580f7 100644 --- a/src/main/java/org/gcube/accounting/aggregator/elaboration/AggregatorManager.java +++ b/src/main/java/org/gcube/accounting/aggregator/elaboration/AggregatorManager.java @@ -29,7 +29,10 @@ public class AggregatorManager { public AggregatorManager(AggregationType aggregationType, boolean restartFromLastAggregationDate, Date aggregationStartDate, Date aggregationEndDate) throws Exception { this.aggregationType = aggregationType; - this.aggregationStartDate = Utility.sanitizeDate(aggregationType, aggregationStartDate); + + if(aggregationStartDate!=null) { + this.aggregationStartDate = Utility.sanitizeDate(aggregationType, aggregationStartDate); + } this.aggregationEndDate = aggregationEndDate; @@ -63,7 +66,7 @@ public class AggregatorManager { return aggregationStatus; } - public void elaborate(Date persistStartTime, Date persistEndTime, String recordType) + public void elaborate(String recordType) throws Exception { AggregationStatus aggregationStatus = null; @@ -95,7 +98,7 @@ public class AggregatorManager { return; } - Elaborator elaborator = new Elaborator(aggregationStatus, persistStartTime, persistEndTime); + Elaborator elaborator = new Elaborator(aggregationStatus); elaborator.elaborate(forceEarlyAggregation, forceRerun, forceRestart); } diff --git a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java index ffc6c6c..e2bd7d1 100644 --- a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java +++ b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java @@ -29,13 +29,11 @@ public class Elaborator { public final static String AGGREGATED_SUFFIX = ".aggregated.json"; protected AggregationStatus aggregationStatus; - protected final Date persistStartTime; - protected final Date persistEndTime; +// protected final Date persistStartTime; +// protected final Date persistEndTime; - public Elaborator(AggregationStatus aggregationStatus, Date persistStartTime, Date persistEndTime) throws Exception { + public Elaborator(AggregationStatus aggregationStatus) throws Exception { this.aggregationStatus = aggregationStatus; - this.persistStartTime = persistStartTime; - this.persistEndTime = persistEndTime; } public boolean isAggregationAllowed(){ @@ -168,6 +166,9 @@ public class Elaborator { Aggregator aggregator = new Aggregator(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile); aggregator.aggregate(); + Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); + persist.recover(); + // Calendar now = Utility.getUTCCalendarInstance(); // /* @@ -176,8 +177,8 @@ public class Elaborator { // * before midnight and the second after midnight (so in the next day). // */ // if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) { - Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); - persist.recover(); +// Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); +// persist.recover(); // }else{ // logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime)); // } diff --git a/src/main/java/org/gcube/accounting/aggregator/elaboration/RecoveryManager.java b/src/main/java/org/gcube/accounting/aggregator/elaboration/RecoveryManager.java index 0a47d7c..62493bf 100644 --- a/src/main/java/org/gcube/accounting/aggregator/elaboration/RecoveryManager.java +++ b/src/main/java/org/gcube/accounting/aggregator/elaboration/RecoveryManager.java @@ -15,8 +15,9 @@ public class RecoveryManager { private static Logger logger = LoggerFactory.getLogger(RecoveryManager.class); - protected final Date persistStartTime; - protected final Date persistEndTime; +// protected final Date persistStartTime; +// protected final Date persistEndTime; + protected final Date aggregationStartDate; protected final Date aggregationEndDate; @@ -24,10 +25,10 @@ public class RecoveryManager { protected AggregationType aggregationType; protected boolean forceRestart; - public RecoveryManager(Date persistStartTime, Date persistEndTime, Date aggregationStartDate, Date aggregationEndDate){ + public RecoveryManager(Date aggregationStartDate, Date aggregationEndDate){ super(); - this.persistStartTime = persistStartTime; - this.persistEndTime = persistEndTime; +// this.persistStartTime = persistStartTime; +// this.persistEndTime = persistEndTime; this.aggregationStartDate = aggregationStartDate; this.aggregationEndDate = aggregationEndDate; this.forceRestart = false; @@ -46,7 +47,7 @@ public class RecoveryManager { aggregationInfo.getAggregationStartDate()); logger.info("Going to Recover unterminated elaboration {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); - Elaborator elaborator = new Elaborator(aggregationStatus, persistStartTime, persistEndTime); + Elaborator elaborator = new Elaborator(aggregationStatus); elaborator.elaborate(true, true, forceRestart); } diff --git a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java index 28134a5..23afcd2 100644 --- a/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java +++ b/src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java @@ -303,7 +303,7 @@ public class PostgreSQLConnector extends AccountingPersistenceQueryPostgreSQL { } stringBuffer.append(" ORDER BY "); - stringBuffer.append("last_update_time DESC LIMIT 1"); + stringBuffer.append("aggregation_start_date DESC LIMIT 1"); Connection connection = getConnection(); Statement statement = connection.createStatement(); diff --git a/src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java b/src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java index 2548a9c..4e34f9a 100644 --- a/src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java +++ b/src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java @@ -1,7 +1,6 @@ package org.gcube.accounting.aggregator.plugin; import java.text.DateFormat; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Map; @@ -72,15 +71,10 @@ public class AccountingAggregatorPlugin extends Plugin { */ public static final String ELABORATION_TYPE_INPUT_PARAMETER = "elaborationType"; - /** - * Start Day Time in UTC when the plugin is allowed to write in buckets - */ - public static final String PERSIST_START_TIME_INPUT_PARAMETER = "persistStartTime"; - - public static final String PERSIST_END_TIME_INPUT_PARAMETER = "persistEndTime"; - - public static final String PERSIST_TIME_DATE_FORMAT_PATTERN = "HH:mm"; - public static final DateFormat PERSIST_TIME_DATE_FORMAT; +// public static final String PERSIST_START_TIME_INPUT_PARAMETER = "persistStartTime"; +// public static final String PERSIST_END_TIME_INPUT_PARAMETER = "persistEndTime"; +// public static final String PERSIST_TIME_DATE_FORMAT_PATTERN = "HH:mm"; +// public static final DateFormat PERSIST_TIME_DATE_FORMAT; public static final String LOCAL_TIME_DATE_FORMAT_PATTERN = "HH:mm Z"; public static final DateFormat LOCAL_TIME_DATE_FORMAT; @@ -91,7 +85,7 @@ public class AccountingAggregatorPlugin extends Plugin { AGGREGATION_START_DATE_DATE_FORMAT = Utility.getUTCDateFormat(AGGREGATION_START_DATE_DATE_FORMAT_PATTERN); AGGREGATION_START_END_DATE_UTC_DATE_FORMAT = Utility.getUTCDateFormat(AGGREGATION_START_END_DATE_UTC_DATE_FORMAT_PATTERN); - PERSIST_TIME_DATE_FORMAT = new SimpleDateFormat(PERSIST_TIME_DATE_FORMAT_PATTERN); +// PERSIST_TIME_DATE_FORMAT = new SimpleDateFormat(PERSIST_TIME_DATE_FORMAT_PATTERN); LOCAL_TIME_DATE_FORMAT = new SimpleDateFormat(LOCAL_TIME_DATE_FORMAT_PATTERN); } @@ -101,19 +95,19 @@ public class AccountingAggregatorPlugin extends Plugin { RegexRulesAggregator.getInstance(); } - private Date getPersistTime(Map inputs, String parameterName) throws ParseException{ - Date persistTime = null; - if (inputs.containsKey(parameterName)) { - String persistTimeString = (String) inputs.get(parameterName); - persistTime = Utility.getPersistTimeDate(persistTimeString); - } - - if(persistTime==null){ - throw new IllegalArgumentException("Please set a valid '" + parameterName +"' by using " + PERSIST_TIME_DATE_FORMAT_PATTERN + " format."); - } - - return persistTime; - } +// private Date getPersistTime(Map inputs, String parameterName) throws ParseException{ +// Date persistTime = null; +// if (inputs.containsKey(parameterName)) { +// String persistTimeString = (String) inputs.get(parameterName); +// persistTime = Utility.getPersistTimeDate(persistTimeString); +// } +// +// if(persistTime==null){ +// throw new IllegalArgumentException("Please set a valid '" + parameterName +"' by using " + PERSIST_TIME_DATE_FORMAT_PATTERN + " format."); +// } +// +// return persistTime; +// } /** {@inheritDoc} */ @SuppressWarnings("unchecked") @@ -127,9 +121,6 @@ public class AccountingAggregatorPlugin extends Plugin { boolean restartFromLastAggregationDate = false; ElaborationType elaborationType = ElaborationType.AGGREGATE; - Date persistStartTime = null; - Date persistEndTime = null; - String recordType = null; Class usageRecordClass = null; @@ -145,9 +136,8 @@ public class AccountingAggregatorPlugin extends Plugin { elaborationType = ElaborationType.valueOf((String) inputs.get(ELABORATION_TYPE_INPUT_PARAMETER)); } - persistStartTime = getPersistTime(inputs, PERSIST_START_TIME_INPUT_PARAMETER); - - persistEndTime = getPersistTime(inputs, PERSIST_END_TIME_INPUT_PARAMETER); +// Date persistStartTime = getPersistTime(inputs, PERSIST_START_TIME_INPUT_PARAMETER); +// Date persistEndTime = getPersistTime(inputs, PERSIST_END_TIME_INPUT_PARAMETER); if (inputs.containsKey(AGGREGATION_START_DATE_INPUT_PARAMETER)) { String aggregationStartDateString = (String) inputs.get(AGGREGATION_START_DATE_INPUT_PARAMETER); @@ -201,12 +191,14 @@ public class AccountingAggregatorPlugin extends Plugin { aggregatorManager.setForceEarlyAggregation(forceEarlyAggregation); aggregatorManager.setForceRerun(forceRerun); aggregatorManager.setForceRestart(forceRestart); - aggregatorManager.elaborate(persistStartTime, persistEndTime, recordType); - + // aggregatorManager.elaborate(persistStartTime, persistEndTime, recordType); + aggregatorManager.elaborate(recordType); + break; case RECOVERY: - RecoveryManager recoveryManager = new RecoveryManager(persistStartTime, persistEndTime, aggregationStartDate, aggregationEndDate); + // RecoveryManager recoveryManager = new RecoveryManager(persistStartTime, persistEndTime, aggregationStartDate, aggregationEndDate); + RecoveryManager recoveryManager = new RecoveryManager(aggregationStartDate, aggregationEndDate); recoveryManager.setForceRestart(forceRestart); recoveryManager.setRecordType(recordType); recoveryManager.setAggregationType(aggregationType); diff --git a/src/main/java/org/gcube/accounting/aggregator/utility/Utility.java b/src/main/java/org/gcube/accounting/aggregator/utility/Utility.java index 65979a9..faad1d2 100644 --- a/src/main/java/org/gcube/accounting/aggregator/utility/Utility.java +++ b/src/main/java/org/gcube/accounting/aggregator/utility/Utility.java @@ -77,17 +77,17 @@ public class Utility { LOCALE_DATE_FORMAT = new SimpleDateFormat(LOCALE_FORMAT_PATTERN); } - public static String getPersistTimeParameter(int hour, int minute) { - // Used from Clients. Not in UTC but in locale - Calendar persistEndTime = Calendar.getInstance(); - persistEndTime.set(Calendar.HOUR_OF_DAY, hour); - persistEndTime.set(Calendar.MINUTE, minute); - - String persistEndTimeParameter = AccountingAggregatorPlugin.PERSIST_TIME_DATE_FORMAT - .format(persistEndTime.getTime()); - - return persistEndTimeParameter; - } +// public static String getPersistTimeParameter(int hour, int minute) { +// // Used from Clients. Not in UTC but in locale +// Calendar persistEndTime = Calendar.getInstance(); +// persistEndTime.set(Calendar.HOUR_OF_DAY, hour); +// persistEndTime.set(Calendar.MINUTE, minute); +// +// String persistEndTimeParameter = AccountingAggregatorPlugin.PERSIST_TIME_DATE_FORMAT +// .format(persistEndTime.getTime()); +// +// return persistEndTimeParameter; +// } public static Date getPersistTimeDate(String persistTimeString) throws ParseException{ Date date = new Date();