gCubeSystem
/
accounting-aggregator-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Removed no more needed persist start and end time check

This commit is contained in:
Luca Frosini 2021-11-11 14:29:09 +01:00
parent 83b63af502
commit 3a90441a22
6 changed files with 58 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/org/gcube/accounting/aggregator/elaboration/AggregatorManager.java
View File

@ -29,7 +29,10 @@ public class AggregatorManager {
public AggregatorManager(AggregationType aggregationType, boolean restartFromLastAggregationDate,
Date aggregationStartDate, Date aggregationEndDate) throws Exception {
this.aggregationType = aggregationType;
if(aggregationStartDate!=null) {
this.aggregationStartDate = Utility.sanitizeDate(aggregationType, aggregationStartDate);
}
this.aggregationEndDate = aggregationEndDate;
@ -63,7 +66,7 @@ public class AggregatorManager {
return aggregationStatus;
}
public void elaborate(Date persistStartTime, Date persistEndTime, String recordType)
public void elaborate(String recordType)
throws Exception {
AggregationStatus aggregationStatus = null;
@ -95,7 +98,7 @@ public class AggregatorManager {
return;
}
Elaborator elaborator = new Elaborator(aggregationStatus, persistStartTime, persistEndTime);
Elaborator elaborator = new Elaborator(aggregationStatus);
elaborator.elaborate(forceEarlyAggregation, forceRerun, forceRestart);
}

15
src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java
View File

@ -29,13 +29,11 @@ public class Elaborator {
public final static String AGGREGATED_SUFFIX = ".aggregated.json";
protected AggregationStatus aggregationStatus;
protected final Date persistStartTime;
protected final Date persistEndTime;
// protected final Date persistStartTime;
// protected final Date persistEndTime;
public Elaborator(AggregationStatus aggregationStatus, Date persistStartTime, Date persistEndTime) throws Exception {
public Elaborator(AggregationStatus aggregationStatus) throws Exception {
this.aggregationStatus = aggregationStatus;
this.persistStartTime = persistStartTime;
this.persistEndTime = persistEndTime;
}
public boolean isAggregationAllowed(){
@ -168,6 +166,9 @@ public class Elaborator {
Aggregator aggregator = new Aggregator(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile);
aggregator.aggregate();
Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
persist.recover();
// Calendar now = Utility.getUTCCalendarInstance();
// /*
@ -176,8 +177,8 @@ public class Elaborator {
// * before midnight and the second after midnight (so in the next day).
// */
// if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) {
Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
persist.recover();
// Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
// persist.recover();
// }else{
// logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime));
// }

13
src/main/java/org/gcube/accounting/aggregator/elaboration/RecoveryManager.java
View File

@ -15,8 +15,9 @@ public class RecoveryManager {
private static Logger logger = LoggerFactory.getLogger(RecoveryManager.class);
protected final Date persistStartTime;
protected final Date persistEndTime;
// protected final Date persistStartTime;
// protected final Date persistEndTime;
protected final Date aggregationStartDate;
protected final Date aggregationEndDate;
@ -24,10 +25,10 @@ public class RecoveryManager {
protected AggregationType aggregationType;
protected boolean forceRestart;
public RecoveryManager(Date persistStartTime, Date persistEndTime, Date aggregationStartDate, Date aggregationEndDate){
public RecoveryManager(Date aggregationStartDate, Date aggregationEndDate){
super();
this.persistStartTime = persistStartTime;
this.persistEndTime = persistEndTime;
// this.persistStartTime = persistStartTime;
// this.persistEndTime = persistEndTime;
this.aggregationStartDate = aggregationStartDate;
this.aggregationEndDate = aggregationEndDate;
this.forceRestart = false;
@ -46,7 +47,7 @@ public class RecoveryManager {
aggregationInfo.getAggregationStartDate());
logger.info("Going to Recover unterminated elaboration {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
Elaborator elaborator = new Elaborator(aggregationStatus, persistStartTime, persistEndTime);
Elaborator elaborator = new Elaborator(aggregationStatus);
elaborator.elaborate(true, true, forceRestart);
}

2
src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java
View File

@ -303,7 +303,7 @@ public class PostgreSQLConnector extends AccountingPersistenceQueryPostgreSQL {
}
stringBuffer.append(" ORDER BY ");
stringBuffer.append("last_update_time DESC LIMIT 1");
stringBuffer.append("aggregation_start_date DESC LIMIT 1");
Connection connection = getConnection();
Statement statement = connection.createStatement();

56
src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java
View File

@ -1,7 +1,6 @@
package org.gcube.accounting.aggregator.plugin;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Map;
@ -72,15 +71,10 @@ public class AccountingAggregatorPlugin extends Plugin {
*/
public static final String ELABORATION_TYPE_INPUT_PARAMETER = "elaborationType";
/**
* Start Day Time in UTC when the plugin is allowed to write in buckets
*/
public static final String PERSIST_START_TIME_INPUT_PARAMETER = "persistStartTime";
public static final String PERSIST_END_TIME_INPUT_PARAMETER = "persistEndTime";
public static final String PERSIST_TIME_DATE_FORMAT_PATTERN = "HH:mm";
public static final DateFormat PERSIST_TIME_DATE_FORMAT;
// public static final String PERSIST_START_TIME_INPUT_PARAMETER = "persistStartTime";
// public static final String PERSIST_END_TIME_INPUT_PARAMETER = "persistEndTime";
// public static final String PERSIST_TIME_DATE_FORMAT_PATTERN = "HH:mm";
// public static final DateFormat PERSIST_TIME_DATE_FORMAT;
public static final String LOCAL_TIME_DATE_FORMAT_PATTERN = "HH:mm Z";
public static final DateFormat LOCAL_TIME_DATE_FORMAT;
@ -91,7 +85,7 @@ public class AccountingAggregatorPlugin extends Plugin {
AGGREGATION_START_DATE_DATE_FORMAT = Utility.getUTCDateFormat(AGGREGATION_START_DATE_DATE_FORMAT_PATTERN);
AGGREGATION_START_END_DATE_UTC_DATE_FORMAT = Utility.getUTCDateFormat(AGGREGATION_START_END_DATE_UTC_DATE_FORMAT_PATTERN);
PERSIST_TIME_DATE_FORMAT = new SimpleDateFormat(PERSIST_TIME_DATE_FORMAT_PATTERN);
// PERSIST_TIME_DATE_FORMAT = new SimpleDateFormat(PERSIST_TIME_DATE_FORMAT_PATTERN);
LOCAL_TIME_DATE_FORMAT = new SimpleDateFormat(LOCAL_TIME_DATE_FORMAT_PATTERN);
}
@ -101,19 +95,19 @@ public class AccountingAggregatorPlugin extends Plugin {
RegexRulesAggregator.getInstance();
}
private Date getPersistTime(Map<String, Object> inputs, String parameterName) throws ParseException{
Date persistTime = null;
if (inputs.containsKey(parameterName)) {
String persistTimeString = (String) inputs.get(parameterName);
persistTime = Utility.getPersistTimeDate(persistTimeString);
}
if(persistTime==null){
throw new IllegalArgumentException("Please set a valid '" + parameterName +"' by using " + PERSIST_TIME_DATE_FORMAT_PATTERN + " format.");
}
return persistTime;
}
// private Date getPersistTime(Map<String, Object> inputs, String parameterName) throws ParseException{
// Date persistTime = null;
// if (inputs.containsKey(parameterName)) {
// String persistTimeString = (String) inputs.get(parameterName);
// persistTime = Utility.getPersistTimeDate(persistTimeString);
// }
//
// if(persistTime==null){
// throw new IllegalArgumentException("Please set a valid '" + parameterName +"' by using " + PERSIST_TIME_DATE_FORMAT_PATTERN + " format.");
// }
//
// return persistTime;
// }
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@ -127,9 +121,6 @@ public class AccountingAggregatorPlugin extends Plugin {
boolean restartFromLastAggregationDate = false;
ElaborationType elaborationType = ElaborationType.AGGREGATE;
Date persistStartTime = null;
Date persistEndTime = null;
String recordType = null;
Class<? extends UsageRecord> usageRecordClass = null;
@ -145,9 +136,8 @@ public class AccountingAggregatorPlugin extends Plugin {
elaborationType = ElaborationType.valueOf((String) inputs.get(ELABORATION_TYPE_INPUT_PARAMETER));
}
persistStartTime = getPersistTime(inputs, PERSIST_START_TIME_INPUT_PARAMETER);
persistEndTime = getPersistTime(inputs, PERSIST_END_TIME_INPUT_PARAMETER);
// Date persistStartTime = getPersistTime(inputs, PERSIST_START_TIME_INPUT_PARAMETER);
// Date persistEndTime = getPersistTime(inputs, PERSIST_END_TIME_INPUT_PARAMETER);
if (inputs.containsKey(AGGREGATION_START_DATE_INPUT_PARAMETER)) {
String aggregationStartDateString = (String) inputs.get(AGGREGATION_START_DATE_INPUT_PARAMETER);
@ -201,12 +191,14 @@ public class AccountingAggregatorPlugin extends Plugin {
aggregatorManager.setForceEarlyAggregation(forceEarlyAggregation);
aggregatorManager.setForceRerun(forceRerun);
aggregatorManager.setForceRestart(forceRestart);
aggregatorManager.elaborate(persistStartTime, persistEndTime, recordType);
// aggregatorManager.elaborate(persistStartTime, persistEndTime, recordType);
aggregatorManager.elaborate(recordType);
break;
case RECOVERY:
RecoveryManager recoveryManager = new RecoveryManager(persistStartTime, persistEndTime, aggregationStartDate, aggregationEndDate);
// RecoveryManager recoveryManager = new RecoveryManager(persistStartTime, persistEndTime, aggregationStartDate, aggregationEndDate);
RecoveryManager recoveryManager = new RecoveryManager(aggregationStartDate, aggregationEndDate);
recoveryManager.setForceRestart(forceRestart);
recoveryManager.setRecordType(recordType);
recoveryManager.setAggregationType(aggregationType);

22
src/main/java/org/gcube/accounting/aggregator/utility/Utility.java
View File

@ -77,17 +77,17 @@ public class Utility {
LOCALE_DATE_FORMAT = new SimpleDateFormat(LOCALE_FORMAT_PATTERN);
}
public static String getPersistTimeParameter(int hour, int minute) {
// Used from Clients. Not in UTC but in locale
Calendar persistEndTime = Calendar.getInstance();
persistEndTime.set(Calendar.HOUR_OF_DAY, hour);
persistEndTime.set(Calendar.MINUTE, minute);
String persistEndTimeParameter = AccountingAggregatorPlugin.PERSIST_TIME_DATE_FORMAT
.format(persistEndTime.getTime());
return persistEndTimeParameter;
}
// public static String getPersistTimeParameter(int hour, int minute) {
// // Used from Clients. Not in UTC but in locale
// Calendar persistEndTime = Calendar.getInstance();
// persistEndTime.set(Calendar.HOUR_OF_DAY, hour);
// persistEndTime.set(Calendar.MINUTE, minute);
//
// String persistEndTimeParameter = AccountingAggregatorPlugin.PERSIST_TIME_DATE_FORMAT
// .format(persistEndTime.getTime());
//
// return persistEndTimeParameter;
// }
public static Date getPersistTimeDate(String persistTimeString) throws ParseException{
Date date = new Date();