Added estimated row to be aggregated as info in logs
This commit is contained in:
parent
a80d2b496a
commit
2c927dda9d
|
@ -46,8 +46,13 @@ public class Aggregator {
|
||||||
protected final File originalRecordsbackupFile;
|
protected final File originalRecordsbackupFile;
|
||||||
protected final File aggregateRecordsBackupFile;
|
protected final File aggregateRecordsBackupFile;
|
||||||
protected final File malformedRecordsFile;
|
protected final File malformedRecordsFile;
|
||||||
|
|
||||||
|
protected int estimatedRecordsNumber;
|
||||||
|
protected int originalRecordsNumber;
|
||||||
protected int malformedRecordNumber;
|
protected int malformedRecordNumber;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
protected ObjectMapper objectMapper;
|
protected ObjectMapper objectMapper;
|
||||||
|
|
||||||
protected Calendar startTime;
|
protected Calendar startTime;
|
||||||
|
@ -73,6 +78,9 @@ public class Aggregator {
|
||||||
startTime = Utility.getUTCCalendarInstance();
|
startTime = Utility.getUTCCalendarInstance();
|
||||||
|
|
||||||
AggregatorPersistenceSrc aggregatorPersistenceSrc = AggregatorPersistenceFactory.getAggregatorPersistenceSrc();
|
AggregatorPersistenceSrc aggregatorPersistenceSrc = AggregatorPersistenceFactory.getAggregatorPersistenceSrc();
|
||||||
|
|
||||||
|
estimatedRecordsNumber = aggregatorPersistenceSrc.getEstimatedRecordRecordToBeAggregated(aggregationStatus);
|
||||||
|
logger.info("Estimated records to be aggregated are {}", estimatedRecordsNumber);
|
||||||
ResultSet resultSet = aggregatorPersistenceSrc.getResultSetOfRecordToBeAggregated(aggregationStatus);
|
ResultSet resultSet = aggregatorPersistenceSrc.getResultSetOfRecordToBeAggregated(aggregationStatus);
|
||||||
|
|
||||||
retrieveAndAggregate(resultSet);
|
retrieveAndAggregate(resultSet);
|
||||||
|
@ -84,7 +92,7 @@ public class Aggregator {
|
||||||
private static final String SIMPLE = "Simple";
|
private static final String SIMPLE = "Simple";
|
||||||
|
|
||||||
|
|
||||||
protected int elaborateRow(ObjectNode content, AggregatorBuffer aggregatorBuffer, int originalRecordsCounter) throws Exception {
|
protected void elaborateRow(ObjectNode content, AggregatorBuffer aggregatorBuffer) throws Exception {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
if(content.has(USAGE_RECORD_TYPE)){
|
if(content.has(USAGE_RECORD_TYPE)){
|
||||||
|
@ -168,23 +176,22 @@ public class Aggregator {
|
||||||
aggregateRow(aggregatorBuffer, record);
|
aggregateRow(aggregatorBuffer, record);
|
||||||
}
|
}
|
||||||
|
|
||||||
++originalRecordsCounter;
|
++originalRecordsNumber;
|
||||||
if(originalRecordsCounter%1000==0){
|
if(originalRecordsNumber%1000==0){
|
||||||
int aggregatedRecordsNumber = 0;
|
int aggregatedRecordsNumber = 0;
|
||||||
if(!skipAggregation) {
|
if(!skipAggregation) {
|
||||||
aggregatedRecordsNumber = aggregatorBuffer.getAggregatedRecords().size();
|
aggregatedRecordsNumber = aggregatorBuffer.getAggregatedRecords().size();
|
||||||
}else {
|
}else {
|
||||||
aggregatedRecordsNumber = originalRecordsCounter;
|
aggregatedRecordsNumber = originalRecordsNumber;
|
||||||
}
|
}
|
||||||
int diff = originalRecordsCounter - aggregatedRecordsNumber;
|
int diff = originalRecordsNumber - aggregatedRecordsNumber;
|
||||||
float percentage = (100 * diff) / originalRecordsCounter;
|
float percentage = (100 * diff) / originalRecordsNumber;
|
||||||
logger.info("{} At the moment, the elaborated original records are {}. The Aggregated records are {}. Difference {}. We are recovering {}% of Documents",
|
logger.info("{} At the moment, the elaborated original records are {} (Total Estimated Number is {}). The Aggregated records are {}. Difference {}. We are recovering {}% of Records",
|
||||||
aggregationStatus.getAggregationInfo(), originalRecordsCounter, aggregatedRecordsNumber, diff, percentage);
|
aggregationStatus.getAggregationInfo(), originalRecordsNumber, estimatedRecordsNumber, aggregatedRecordsNumber, diff, percentage);
|
||||||
}
|
}
|
||||||
|
|
||||||
Utility.printLine(originalRecordsbackupFile, record);
|
Utility.printLine(originalRecordsbackupFile, record);
|
||||||
|
|
||||||
return originalRecordsCounter;
|
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
|
@ -236,8 +243,8 @@ public class Aggregator {
|
||||||
|
|
||||||
Set<String> requiredFields = clz.newInstance().getRequiredFields();
|
Set<String> requiredFields = clz.newInstance().getRequiredFields();
|
||||||
|
|
||||||
|
originalRecordsNumber = 0;
|
||||||
malformedRecordNumber = 0;
|
malformedRecordNumber = 0;
|
||||||
int originalRecordsCounter = 0;
|
|
||||||
while (resultSet.next()) {
|
while (resultSet.next()) {
|
||||||
for(int i=1; i<=MAX_RETRY; i++){
|
for(int i=1; i<=MAX_RETRY; i++){
|
||||||
try {
|
try {
|
||||||
|
@ -267,7 +274,7 @@ public class Aggregator {
|
||||||
addProperty(objectNode, recordField, obj);
|
addProperty(objectNode, recordField, obj);
|
||||||
}
|
}
|
||||||
|
|
||||||
originalRecordsCounter = elaborateRow(objectNode, aggregatorBuffer, originalRecordsCounter);
|
elaborateRow(objectNode, aggregatorBuffer);
|
||||||
TimeUnit.MILLISECONDS.sleep(3);
|
TimeUnit.MILLISECONDS.sleep(3);
|
||||||
break;
|
break;
|
||||||
}catch (RuntimeException e) {
|
}catch (RuntimeException e) {
|
||||||
|
@ -304,10 +311,10 @@ public class Aggregator {
|
||||||
|
|
||||||
aggregateRecordsBackupFileTmp.renameTo(aggregateRecordsBackupFile);
|
aggregateRecordsBackupFileTmp.renameTo(aggregateRecordsBackupFile);
|
||||||
|
|
||||||
aggregationStatus.setRecordNumbers(originalRecordsCounter, aggregatedRecords.size(), malformedRecordNumber);
|
aggregationStatus.setRecordNumbers(originalRecordsNumber, aggregatedRecords.size(), malformedRecordNumber);
|
||||||
}else {
|
}else {
|
||||||
Files.copy(originalRecordsbackupFile.toPath(), aggregateRecordsBackupFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
Files.copy(originalRecordsbackupFile.toPath(), aggregateRecordsBackupFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||||
aggregationStatus.setRecordNumbers(originalRecordsCounter, originalRecordsCounter, malformedRecordNumber);
|
aggregationStatus.setRecordNumbers(originalRecordsNumber, originalRecordsNumber, malformedRecordNumber);
|
||||||
}
|
}
|
||||||
|
|
||||||
aggregationStatus.setAggregationState(AggregationState.AGGREGATED, startTime, true);
|
aggregationStatus.setAggregationState(AggregationState.AGGREGATED, startTime, true);
|
||||||
|
|
|
@ -30,7 +30,7 @@ public class DeleteDocument extends DocumentElaboration {
|
||||||
if(aggregatorPersistenceSrc.isBulkDeleteAllowed()) {
|
if(aggregatorPersistenceSrc.isBulkDeleteAllowed()) {
|
||||||
arrayNode.add(jsonNode);
|
arrayNode.add(jsonNode);
|
||||||
aggregatorPersistenceSrc.deleteRecord(jsonNode);
|
aggregatorPersistenceSrc.deleteRecord(jsonNode);
|
||||||
if(arrayNode.size()>=MAX_ROWS_PER_STEP) {
|
if(arrayNode.size()>=effectiveMaxRowPerStep) {
|
||||||
aggregatorPersistenceSrc.deleteRecords(arrayNode);
|
aggregatorPersistenceSrc.deleteRecords(arrayNode);
|
||||||
arrayNode = DSMapper.getObjectMapper().createArrayNode();
|
arrayNode = DSMapper.getObjectMapper().createArrayNode();
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,6 +37,8 @@ public abstract class DocumentElaboration {
|
||||||
|
|
||||||
protected Calendar startTime;
|
protected Calendar startTime;
|
||||||
|
|
||||||
|
protected int effectiveMaxRowPerStep;
|
||||||
|
|
||||||
protected DocumentElaboration(AggregationStatus statusManager, AggregationState finalAggregationState, File file, int rowToBeElaborated) {
|
protected DocumentElaboration(AggregationStatus statusManager, AggregationState finalAggregationState, File file, int rowToBeElaborated) {
|
||||||
this.aggregationStatus = statusManager;
|
this.aggregationStatus = statusManager;
|
||||||
this.finalAggregationState = finalAggregationState;
|
this.finalAggregationState = finalAggregationState;
|
||||||
|
@ -58,9 +60,9 @@ public abstract class DocumentElaboration {
|
||||||
|
|
||||||
logger.info("{} - Going to elaborate {} rows", aggregationStatus.getAggregationInfo(), rowToBeElaborated);
|
logger.info("{} - Going to elaborate {} rows", aggregationStatus.getAggregationInfo(), rowToBeElaborated);
|
||||||
|
|
||||||
int numberOfRowsForEachRecoveryPoint = (rowToBeElaborated / 10) + 1;
|
effectiveMaxRowPerStep = (rowToBeElaborated / 10) + 1;
|
||||||
if(numberOfRowsForEachRecoveryPoint>MAX_ROWS_PER_STEP) {
|
if(effectiveMaxRowPerStep>MAX_ROWS_PER_STEP) {
|
||||||
numberOfRowsForEachRecoveryPoint = MAX_ROWS_PER_STEP;
|
effectiveMaxRowPerStep = MAX_ROWS_PER_STEP;
|
||||||
}
|
}
|
||||||
|
|
||||||
currentlyElaborated = 0;
|
currentlyElaborated = 0;
|
||||||
|
@ -104,7 +106,7 @@ public abstract class DocumentElaboration {
|
||||||
}
|
}
|
||||||
|
|
||||||
++currentlyElaborated;
|
++currentlyElaborated;
|
||||||
if(currentlyElaborated % numberOfRowsForEachRecoveryPoint == 0) {
|
if(currentlyElaborated % effectiveMaxRowPerStep == 0) {
|
||||||
if(currentlyElaborated>=restartFrom) {
|
if(currentlyElaborated>=restartFrom) {
|
||||||
aggregationStatus.setRestartFrom(currentlyElaborated, true);
|
aggregationStatus.setRestartFrom(currentlyElaborated, true);
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,4 +23,6 @@ public interface AggregatorPersistenceSrc extends AggregatorPersistence {
|
||||||
*/
|
*/
|
||||||
public void deleteRecords(ArrayNode array) throws UnsupportedOperationException, Exception;
|
public void deleteRecords(ArrayNode array) throws UnsupportedOperationException, Exception;
|
||||||
|
|
||||||
|
public int getEstimatedRecordRecordToBeAggregated(AggregationStatus aggregationStatus) throws Exception;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -562,6 +562,39 @@ public class PostgreSQLConnector extends PersistencePostgreSQL implements Aggreg
|
||||||
return resultSet;
|
return resultSet;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getEstimatedRecordRecordToBeAggregated(AggregationStatus aggregationStatus) throws Exception {
|
||||||
|
AggregationInfo aggregationInfo = aggregationStatus.getAggregationInfo();
|
||||||
|
|
||||||
|
String tableName = RecordToDBFields.getKey(aggregationInfo.getRecordType());
|
||||||
|
|
||||||
|
String startTimeColumnName = RecordToDBFields.getKey(AggregatedUsageRecord.START_TIME);
|
||||||
|
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append("SELECT COUNT(*)");
|
||||||
|
stringBuffer.append(" FROM ");
|
||||||
|
stringBuffer.append(tableName);
|
||||||
|
stringBuffer.append(" WHERE ");
|
||||||
|
stringBuffer.append(startTimeColumnName);
|
||||||
|
stringBuffer.append(" >= ");
|
||||||
|
stringBuffer.append(getValue(aggregationInfo.getAggregationStartDate()));
|
||||||
|
stringBuffer.append(" AND ");
|
||||||
|
stringBuffer.append(startTimeColumnName);
|
||||||
|
stringBuffer.append(" < ");
|
||||||
|
stringBuffer.append(getValue(aggregationInfo.getAggregationEndDate()));
|
||||||
|
|
||||||
|
Connection connection = getConnection();
|
||||||
|
Statement statement = connection.createStatement();
|
||||||
|
|
||||||
|
String sqlQuery = stringBuffer.toString();
|
||||||
|
|
||||||
|
logger.trace("Going to request the following query: {}", sqlQuery);
|
||||||
|
ResultSet resultSet = statement.executeQuery(sqlQuery);
|
||||||
|
|
||||||
|
resultSet.next();
|
||||||
|
return resultSet.getInt(1);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isBulkDeleteAllowed() {
|
public boolean isBulkDeleteAllowed() {
|
||||||
return true;
|
return true;
|
||||||
|
@ -606,5 +639,4 @@ public class PostgreSQLConnector extends PersistencePostgreSQL implements Aggreg
|
||||||
connection.commit();
|
connection.commit();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -77,7 +77,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Ignore
|
// @Ignore
|
||||||
@Test
|
@Test
|
||||||
public void aggregateAllServiceUsageRecord() throws Exception {
|
public void aggregateAllServiceUsageRecord() throws Exception {
|
||||||
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
||||||
|
@ -86,15 +86,15 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void aggregateMonthlyFirstHalf2022ServiceUsageRecord() throws Exception {
|
public void aggregateServiceUsageRecordDaily() throws Exception {
|
||||||
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
|
||||||
|
|
||||||
boolean forceRestart = true;
|
boolean forceRestart = true;
|
||||||
boolean forceEarlyAggregation = true;
|
boolean forceEarlyAggregation = true;
|
||||||
|
|
||||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
AggregationType aggregationType = AggregationType.DAILY;
|
||||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.JUNE, 1);
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 26);
|
||||||
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.JULY, 1);
|
Calendar end = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 27);
|
||||||
while (aggregationStartCalendar.before(end)) {
|
while (aggregationStartCalendar.before(end)) {
|
||||||
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
||||||
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
||||||
|
@ -111,7 +111,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
||||||
boolean forceEarlyAggregation = true;
|
boolean forceEarlyAggregation = true;
|
||||||
|
|
||||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.NOVEMBER, 1);
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.DECEMBER, 1);
|
||||||
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
|
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
|
||||||
while (aggregationStartCalendar.before(end)) {
|
while (aggregationStartCalendar.before(end)) {
|
||||||
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
||||||
|
@ -129,8 +129,8 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
||||||
boolean forceEarlyAggregation = true;
|
boolean forceEarlyAggregation = true;
|
||||||
|
|
||||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.MAY, 1);
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.SEPTEMBER, 1);
|
||||||
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JULY, 1);
|
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.OCTOBER, 1);
|
||||||
while (aggregationStartCalendar.before(end)) {
|
while (aggregationStartCalendar.before(end)) {
|
||||||
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
||||||
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
||||||
|
@ -142,12 +142,13 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
||||||
@Test
|
@Test
|
||||||
public void aggregateFirstQuarter2022StorageStatusRecord() throws Exception {
|
public void aggregateFirstQuarter2022StorageStatusRecord() throws Exception {
|
||||||
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
|
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
|
||||||
|
|
||||||
boolean forceRestart = true;
|
boolean forceRestart = true;
|
||||||
boolean forceEarlyAggregation = true;
|
boolean forceEarlyAggregation = true;
|
||||||
|
|
||||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.FEBRUARY, 1);
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.JANUARY, 1);
|
||||||
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.APRIL, 1);
|
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.FEBRUARY, 1);
|
||||||
while (aggregationStartCalendar.before(end)) {
|
while (aggregationStartCalendar.before(end)) {
|
||||||
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
||||||
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
||||||
|
@ -164,7 +165,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
||||||
boolean forceEarlyAggregation = true;
|
boolean forceEarlyAggregation = true;
|
||||||
|
|
||||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.APRIL, 1);
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.MAY, 1);
|
||||||
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.JULY, 1);
|
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.JULY, 1);
|
||||||
while (aggregationStartCalendar.before(end)) {
|
while (aggregationStartCalendar.before(end)) {
|
||||||
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
||||||
|
@ -203,7 +204,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
||||||
boolean forceRestart = true;
|
boolean forceRestart = true;
|
||||||
boolean forceEarlyAggregation = true;
|
boolean forceEarlyAggregation = true;
|
||||||
AggregationType aggregationType = AggregationType.DAILY;
|
AggregationType aggregationType = AggregationType.DAILY;
|
||||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 25);
|
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 27);
|
||||||
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
|
||||||
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue