gCubeSystem
/
accounting-aggregator-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Removed no more needed persist start and end time check

This commit is contained in:
Luca Frosini 2021-11-11 14:29:09 +01:00
parent 83b63af502
commit 3a90441a22
6 changed files with 58 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/org/gcube/accounting/aggregator/elaboration/AggregatorManager.java
View File

@ -29,7 +29,10 @@ public class AggregatorManager {
public AggregatorManager(AggregationType aggregationType, boolean restartFromLastAggregationDate, public AggregatorManager(AggregationType aggregationType, boolean restartFromLastAggregationDate,
Date aggregationStartDate, Date aggregationEndDate) throws Exception { Date aggregationStartDate, Date aggregationEndDate) throws Exception {
this.aggregationType = aggregationType; this.aggregationType = aggregationType;
if(aggregationStartDate!=null) {
this.aggregationStartDate = Utility.sanitizeDate(aggregationType, aggregationStartDate); this.aggregationStartDate = Utility.sanitizeDate(aggregationType, aggregationStartDate);
}
this.aggregationEndDate = aggregationEndDate; this.aggregationEndDate = aggregationEndDate;
@ -63,7 +66,7 @@ public class AggregatorManager {
return aggregationStatus; return aggregationStatus;
} }
public void elaborate(Date persistStartTime, Date persistEndTime, String recordType) public void elaborate(String recordType)
throws Exception { throws Exception {
AggregationStatus aggregationStatus = null; AggregationStatus aggregationStatus = null;
@ -95,7 +98,7 @@ public class AggregatorManager {
return; return;
} }
Elaborator elaborator = new Elaborator(aggregationStatus, persistStartTime, persistEndTime); Elaborator elaborator = new Elaborator(aggregationStatus);
elaborator.elaborate(forceEarlyAggregation, forceRerun, forceRestart); elaborator.elaborate(forceEarlyAggregation, forceRerun, forceRestart);
} }

15
src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java
View File

@ -29,13 +29,11 @@ public class Elaborator {
public final static String AGGREGATED_SUFFIX = ".aggregated.json"; public final static String AGGREGATED_SUFFIX = ".aggregated.json";
protected AggregationStatus aggregationStatus; protected AggregationStatus aggregationStatus;
protected final Date persistStartTime; // protected final Date persistStartTime;
protected final Date persistEndTime; // protected final Date persistEndTime;
public Elaborator(AggregationStatus aggregationStatus, Date persistStartTime, Date persistEndTime) throws Exception { public Elaborator(AggregationStatus aggregationStatus) throws Exception {
this.aggregationStatus = aggregationStatus; this.aggregationStatus = aggregationStatus;
this.persistStartTime = persistStartTime;
this.persistEndTime = persistEndTime;
} }
public boolean isAggregationAllowed(){ public boolean isAggregationAllowed(){
@ -168,6 +166,9 @@ public class Elaborator {
Aggregator aggregator = new Aggregator(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile); Aggregator aggregator = new Aggregator(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile);
aggregator.aggregate(); aggregator.aggregate();
Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
persist.recover();
// Calendar now = Utility.getUTCCalendarInstance(); // Calendar now = Utility.getUTCCalendarInstance();
// /* // /*
@ -176,8 +177,8 @@ public class Elaborator {
// * before midnight and the second after midnight (so in the next day). // * before midnight and the second after midnight (so in the next day).
// */ // */
// if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) { // if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) {
Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); // Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
persist.recover(); // persist.recover();
// }else{ // }else{
// logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime)); // logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime));
// } // }

13
src/main/java/org/gcube/accounting/aggregator/elaboration/RecoveryManager.java
View File

@ -15,8 +15,9 @@ public class RecoveryManager {
private static Logger logger = LoggerFactory.getLogger(RecoveryManager.class); private static Logger logger = LoggerFactory.getLogger(RecoveryManager.class);
protected final Date persistStartTime; // protected final Date persistStartTime;
protected final Date persistEndTime; // protected final Date persistEndTime;
protected final Date aggregationStartDate; protected final Date aggregationStartDate;
protected final Date aggregationEndDate; protected final Date aggregationEndDate;
@ -24,10 +25,10 @@ public class RecoveryManager {
protected AggregationType aggregationType; protected AggregationType aggregationType;
protected boolean forceRestart; protected boolean forceRestart;
public RecoveryManager(Date persistStartTime, Date persistEndTime, Date aggregationStartDate, Date aggregationEndDate){ public RecoveryManager(Date aggregationStartDate, Date aggregationEndDate){
super(); super();
this.persistStartTime = persistStartTime; // this.persistStartTime = persistStartTime;
this.persistEndTime = persistEndTime; // this.persistEndTime = persistEndTime;
this.aggregationStartDate = aggregationStartDate; this.aggregationStartDate = aggregationStartDate;
this.aggregationEndDate = aggregationEndDate; this.aggregationEndDate = aggregationEndDate;
this.forceRestart = false; this.forceRestart = false;
@ -46,7 +47,7 @@ public class RecoveryManager {
aggregationInfo.getAggregationStartDate()); aggregationInfo.getAggregationStartDate());
logger.info("Going to Recover unterminated elaboration {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus)); logger.info("Going to Recover unterminated elaboration {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
Elaborator elaborator = new Elaborator(aggregationStatus, persistStartTime, persistEndTime); Elaborator elaborator = new Elaborator(aggregationStatus);
elaborator.elaborate(true, true, forceRestart); elaborator.elaborate(true, true, forceRestart);
} }

2
src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java
View File

@ -303,7 +303,7 @@ public class PostgreSQLConnector extends AccountingPersistenceQueryPostgreSQL {
} }
stringBuffer.append(" ORDER BY "); stringBuffer.append(" ORDER BY ");
stringBuffer.append("last_update_time DESC LIMIT 1"); stringBuffer.append("aggregation_start_date DESC LIMIT 1");
Connection connection = getConnection(); Connection connection = getConnection();
Statement statement = connection.createStatement(); Statement statement = connection.createStatement();

56
src/main/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPlugin.java
View File

@ -1,7 +1,6 @@
package org.gcube.accounting.aggregator.plugin; package org.gcube.accounting.aggregator.plugin;
import java.text.DateFormat; import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.Map; import java.util.Map;
@ -72,15 +71,10 @@ public class AccountingAggregatorPlugin extends Plugin {
*/ */
public static final String ELABORATION_TYPE_INPUT_PARAMETER = "elaborationType"; public static final String ELABORATION_TYPE_INPUT_PARAMETER = "elaborationType";
/** // public static final String PERSIST_START_TIME_INPUT_PARAMETER = "persistStartTime";
* Start Day Time in UTC when the plugin is allowed to write in buckets // public static final String PERSIST_END_TIME_INPUT_PARAMETER = "persistEndTime";
*/ // public static final String PERSIST_TIME_DATE_FORMAT_PATTERN = "HH:mm";
public static final String PERSIST_START_TIME_INPUT_PARAMETER = "persistStartTime"; // public static final DateFormat PERSIST_TIME_DATE_FORMAT;
public static final String PERSIST_END_TIME_INPUT_PARAMETER = "persistEndTime";
public static final String PERSIST_TIME_DATE_FORMAT_PATTERN = "HH:mm";
public static final DateFormat PERSIST_TIME_DATE_FORMAT;
public static final String LOCAL_TIME_DATE_FORMAT_PATTERN = "HH:mm Z"; public static final String LOCAL_TIME_DATE_FORMAT_PATTERN = "HH:mm Z";
public static final DateFormat LOCAL_TIME_DATE_FORMAT; public static final DateFormat LOCAL_TIME_DATE_FORMAT;
@ -91,7 +85,7 @@ public class AccountingAggregatorPlugin extends Plugin {
AGGREGATION_START_DATE_DATE_FORMAT = Utility.getUTCDateFormat(AGGREGATION_START_DATE_DATE_FORMAT_PATTERN); AGGREGATION_START_DATE_DATE_FORMAT = Utility.getUTCDateFormat(AGGREGATION_START_DATE_DATE_FORMAT_PATTERN);
AGGREGATION_START_END_DATE_UTC_DATE_FORMAT = Utility.getUTCDateFormat(AGGREGATION_START_END_DATE_UTC_DATE_FORMAT_PATTERN); AGGREGATION_START_END_DATE_UTC_DATE_FORMAT = Utility.getUTCDateFormat(AGGREGATION_START_END_DATE_UTC_DATE_FORMAT_PATTERN);
PERSIST_TIME_DATE_FORMAT = new SimpleDateFormat(PERSIST_TIME_DATE_FORMAT_PATTERN); // PERSIST_TIME_DATE_FORMAT = new SimpleDateFormat(PERSIST_TIME_DATE_FORMAT_PATTERN);
LOCAL_TIME_DATE_FORMAT = new SimpleDateFormat(LOCAL_TIME_DATE_FORMAT_PATTERN); LOCAL_TIME_DATE_FORMAT = new SimpleDateFormat(LOCAL_TIME_DATE_FORMAT_PATTERN);
} }
@ -101,19 +95,19 @@ public class AccountingAggregatorPlugin extends Plugin {
RegexRulesAggregator.getInstance(); RegexRulesAggregator.getInstance();
} }
private Date getPersistTime(Map<String, Object> inputs, String parameterName) throws ParseException{ // private Date getPersistTime(Map<String, Object> inputs, String parameterName) throws ParseException{
Date persistTime = null; // Date persistTime = null;
if (inputs.containsKey(parameterName)) { // if (inputs.containsKey(parameterName)) {
String persistTimeString = (String) inputs.get(parameterName); // String persistTimeString = (String) inputs.get(parameterName);
persistTime = Utility.getPersistTimeDate(persistTimeString); // persistTime = Utility.getPersistTimeDate(persistTimeString);
} // }
//
if(persistTime==null){ // if(persistTime==null){
throw new IllegalArgumentException("Please set a valid '" + parameterName +"' by using " + PERSIST_TIME_DATE_FORMAT_PATTERN + " format."); // throw new IllegalArgumentException("Please set a valid '" + parameterName +"' by using " + PERSIST_TIME_DATE_FORMAT_PATTERN + " format.");
} // }
//
return persistTime; // return persistTime;
} // }
/** {@inheritDoc} */ /** {@inheritDoc} */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -127,9 +121,6 @@ public class AccountingAggregatorPlugin extends Plugin {
boolean restartFromLastAggregationDate = false; boolean restartFromLastAggregationDate = false;
ElaborationType elaborationType = ElaborationType.AGGREGATE; ElaborationType elaborationType = ElaborationType.AGGREGATE;
Date persistStartTime = null;
Date persistEndTime = null;
String recordType = null; String recordType = null;
Class<? extends UsageRecord> usageRecordClass = null; Class<? extends UsageRecord> usageRecordClass = null;
@ -145,9 +136,8 @@ public class AccountingAggregatorPlugin extends Plugin {
elaborationType = ElaborationType.valueOf((String) inputs.get(ELABORATION_TYPE_INPUT_PARAMETER)); elaborationType = ElaborationType.valueOf((String) inputs.get(ELABORATION_TYPE_INPUT_PARAMETER));
} }
persistStartTime = getPersistTime(inputs, PERSIST_START_TIME_INPUT_PARAMETER); // Date persistStartTime = getPersistTime(inputs, PERSIST_START_TIME_INPUT_PARAMETER);
// Date persistEndTime = getPersistTime(inputs, PERSIST_END_TIME_INPUT_PARAMETER);
persistEndTime = getPersistTime(inputs, PERSIST_END_TIME_INPUT_PARAMETER);
if (inputs.containsKey(AGGREGATION_START_DATE_INPUT_PARAMETER)) { if (inputs.containsKey(AGGREGATION_START_DATE_INPUT_PARAMETER)) {
String aggregationStartDateString = (String) inputs.get(AGGREGATION_START_DATE_INPUT_PARAMETER); String aggregationStartDateString = (String) inputs.get(AGGREGATION_START_DATE_INPUT_PARAMETER);
@ -201,12 +191,14 @@ public class AccountingAggregatorPlugin extends Plugin {
aggregatorManager.setForceEarlyAggregation(forceEarlyAggregation); aggregatorManager.setForceEarlyAggregation(forceEarlyAggregation);
aggregatorManager.setForceRerun(forceRerun); aggregatorManager.setForceRerun(forceRerun);
aggregatorManager.setForceRestart(forceRestart); aggregatorManager.setForceRestart(forceRestart);
aggregatorManager.elaborate(persistStartTime, persistEndTime, recordType); // aggregatorManager.elaborate(persistStartTime, persistEndTime, recordType);
aggregatorManager.elaborate(recordType);
break; break;
case RECOVERY: case RECOVERY:
RecoveryManager recoveryManager = new RecoveryManager(persistStartTime, persistEndTime, aggregationStartDate, aggregationEndDate); // RecoveryManager recoveryManager = new RecoveryManager(persistStartTime, persistEndTime, aggregationStartDate, aggregationEndDate);
RecoveryManager recoveryManager = new RecoveryManager(aggregationStartDate, aggregationEndDate);
recoveryManager.setForceRestart(forceRestart); recoveryManager.setForceRestart(forceRestart);
recoveryManager.setRecordType(recordType); recoveryManager.setRecordType(recordType);
recoveryManager.setAggregationType(aggregationType); recoveryManager.setAggregationType(aggregationType);

22
src/main/java/org/gcube/accounting/aggregator/utility/Utility.java
View File

@ -77,17 +77,17 @@ public class Utility {
LOCALE_DATE_FORMAT = new SimpleDateFormat(LOCALE_FORMAT_PATTERN); LOCALE_DATE_FORMAT = new SimpleDateFormat(LOCALE_FORMAT_PATTERN);
} }
public static String getPersistTimeParameter(int hour, int minute) { // public static String getPersistTimeParameter(int hour, int minute) {
// Used from Clients. Not in UTC but in locale // // Used from Clients. Not in UTC but in locale
Calendar persistEndTime = Calendar.getInstance(); // Calendar persistEndTime = Calendar.getInstance();
persistEndTime.set(Calendar.HOUR_OF_DAY, hour); // persistEndTime.set(Calendar.HOUR_OF_DAY, hour);
persistEndTime.set(Calendar.MINUTE, minute); // persistEndTime.set(Calendar.MINUTE, minute);
//
String persistEndTimeParameter = AccountingAggregatorPlugin.PERSIST_TIME_DATE_FORMAT // String persistEndTimeParameter = AccountingAggregatorPlugin.PERSIST_TIME_DATE_FORMAT
.format(persistEndTime.getTime()); // .format(persistEndTime.getTime());
//
return persistEndTimeParameter; // return persistEndTimeParameter;
} // }
public static Date getPersistTimeDate(String persistTimeString) throws ParseException{ public static Date getPersistTimeDate(String persistTimeString) throws ParseException{
Date date = new Date(); Date date = new Date();