gCubeSystem
/
accounting-aggregator-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added bulk delete to try to speedup the process

This commit is contained in:
Luca Frosini 2024-02-27 12:12:57 +01:00
parent 0c236e845d
commit 3d9fd9d42f
5 changed files with 100 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/main/java/org/gcube/accounting/aggregator/persist/DeleteDocument.java
View File

@ -7,6 +7,7 @@ import org.gcube.accounting.aggregator.persistence.AggregatorPersistenceSrc;
import org.gcube.accounting.aggregator.status.AggregationState;
import org.gcube.accounting.aggregator.status.AggregationStatus;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.documentstore.records.DSMapper;
/**
@ -14,22 +15,39 @@ import org.gcube.documentstore.records.DSMapper;
*/
public class DeleteDocument extends DocumentElaboration {
public DeleteDocument(AggregationStatus aggregationStatus, File file){
protected AggregatorPersistenceSrc aggregatorPersistenceSrc;
protected ArrayNode arrayNode;
public DeleteDocument(AggregationStatus aggregationStatus, File file) throws Exception{
super(aggregationStatus, AggregationState.DELETED, file, aggregationStatus.getOriginalRecordsNumber());
arrayNode = DSMapper.getObjectMapper().createArrayNode();
aggregatorPersistenceSrc = AggregatorPersistenceFactory.getAggregatorPersistenceSrc();
}
@Override
protected void elaborateLine(String line) throws Exception {
JsonNode jsonNode = DSMapper.asJsonNode(line);
String id = jsonNode.get(ID).asText();
logger.trace("Going to delete record with id {}", id);
AggregatorPersistenceSrc aggregatorPersistenceSrc = AggregatorPersistenceFactory.getAggregatorPersistenceSrc();
aggregatorPersistenceSrc.deleteRecord(jsonNode);
if(aggregatorPersistenceSrc.isBulkDeleteAllowed()) {
arrayNode.add(jsonNode);
aggregatorPersistenceSrc.deleteRecord(jsonNode);
if(arrayNode.size()>=MAX_ROWS_PER_STEP) {
aggregatorPersistenceSrc.deleteRecords(arrayNode);
arrayNode = DSMapper.getObjectMapper().createArrayNode();
}
} else {
String id = jsonNode.get(ID).asText();
logger.trace("Going to delete record with id {}", id);
aggregatorPersistenceSrc.deleteRecord(jsonNode);
}
}
@Override
protected void afterElaboration() {
protected void afterElaboration() throws Exception {
// Nothing to do
if(aggregatorPersistenceSrc.isBulkDeleteAllowed() && arrayNode.size()>0) {
aggregatorPersistenceSrc.deleteRecords(arrayNode);
arrayNode = DSMapper.getObjectMapper().createArrayNode();
}
}
}

6
src/main/java/org/gcube/accounting/aggregator/persist/DocumentElaboration.java
View File

@ -26,6 +26,8 @@ public abstract class DocumentElaboration {
public static final int MAX_RETRY = 7;
public static final int MAX_ROWS_PER_STEP = 500;
protected final AggregationStatus aggregationStatus;
protected final File file;
protected final AggregationState finalAggregationState;
@ -57,8 +59,8 @@ public abstract class DocumentElaboration {
logger.info("{} - Going to elaborate {} rows", aggregationStatus.getAggregationInfo(), rowToBeElaborated);
int numberOfRowsForEachRecoveryPoint = (rowToBeElaborated / 10) + 1;
if(numberOfRowsForEachRecoveryPoint>500) {
numberOfRowsForEachRecoveryPoint = 500;
if(numberOfRowsForEachRecoveryPoint>MAX_ROWS_PER_STEP) {
numberOfRowsForEachRecoveryPoint = MAX_ROWS_PER_STEP;
}
currentlyElaborated = 0;

9
src/main/java/org/gcube/accounting/aggregator/persistence/AggregatorPersistenceSrc.java
View File

@ -4,6 +4,7 @@ import java.sql.ResultSet;
import org.gcube.accounting.aggregator.status.AggregationStatus;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
/**
* @author Luca Frosini (ISTI - CNR)
@ -14,4 +15,12 @@ public interface AggregatorPersistenceSrc extends AggregatorPersistence {
public void deleteRecord(JsonNode jsonNode) throws Exception;
public boolean isBulkDeleteAllowed();
/**
* It must be implemented only and only if isBulkDeleteAllowed()
* return true. It must raise UnsupportedOperationException if bulk delete is not allowed
*/
public void deleteRecords(ArrayNode array) throws UnsupportedOperationException, Exception;
}

46
src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java
View File

@ -27,6 +27,7 @@ import org.gcube.accounting.persistence.AccountingPersistenceConfiguration;
import org.gcube.accounting.utility.postgresql.RecordToDBFields;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.documentstore.persistence.PersistencePostgreSQL;
import org.gcube.documentstore.records.DSMapper;
import org.gcube.documentstore.records.Record;
@ -560,5 +561,50 @@ public class PostgreSQLConnector extends PersistencePostgreSQL implements Aggreg
return resultSet;
}
@Override
public boolean isBulkDeleteAllowed() {
return true;
}
@Override
public void deleteRecords(ArrayNode array) throws UnsupportedOperationException, Exception {
if(array.size()<1) {
return;
}
Record record = DSMapper.unmarshal(Record.class, array.get(0).toString());
Class<? extends Record> clz = record.getClass();
String type = RecordToDBMapping.getRecordTypeByClass(clz);
String tableName = RecordToDBFields.getKey(type);
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("DELETE ");
stringBuffer.append("FROM ");
stringBuffer.append(tableName);
stringBuffer.append(" WHERE ");
String id = record.getId();
stringBuffer.append("id = ");
stringBuffer.append(getValue(id));
for(int i=1; i<array.size(); i++) {
stringBuffer.append(" OR ");
id = array.get(i).get(Record.ID).asText();
stringBuffer.append("id = ");
stringBuffer.append(getValue(id));
}
Connection connection = getConnection();
Statement statement = connection.createStatement();
String sqlCommand = stringBuffer.toString();
logger.trace("Going to execute {}", sqlCommand);
statement.execute(sqlCommand);
statement.close();
connection.commit();
}
}

61
src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java
View File

@ -13,7 +13,7 @@ import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.StorageStatusRecord;
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -77,16 +77,14 @@ public class AccountingAggregatorPluginTest extends ContextTest {
}
@JsonIgnore
@Ignore
@Test
public void aggregateAllServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
// aggregateOneShot(recordType);
aggregateEverything(recordType);
aggregateOneShot(recordType);
// aggregateEverything(recordType);
}
// @JsonIgnore
@Test
public void aggregateMonthlyFirstHalf2022ServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
@ -95,7 +93,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.FEBRUARY, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.JUNE, 1);
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.JULY, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
@ -105,7 +103,6 @@ public class AccountingAggregatorPluginTest extends ContextTest {
}
}
// @JsonIgnore
@Test
public void aggregateMonthlySecondHalf2022ServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
@ -114,7 +111,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.JULY, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.NOVEMBER, 1);
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
@ -124,7 +121,6 @@ public class AccountingAggregatorPluginTest extends ContextTest {
}
}
@JsonIgnore
@Test
public void aggregateMonthlyFirstHalf2023ServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
@ -133,7 +129,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.MAY, 1);
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JULY, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
@ -142,37 +138,16 @@ public class AccountingAggregatorPluginTest extends ContextTest {
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
}
}
@JsonIgnore
@Test
public void aggregateMonthlySecondHalf2023ServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.JULY, 1);
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
aggregationStartCalendar = Calendar.getInstance();
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
}
}
// @JsonIgnore
@Test
public void aggregateMonthlyStorageStatusRecord() throws Exception {
public void aggregateFirstQuarter2022StorageStatusRecord() throws Exception {
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.NOVEMBER, 1);
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.JUNE, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.FEBRUARY, 1);
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.APRIL, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
@ -181,17 +156,16 @@ public class AccountingAggregatorPluginTest extends ContextTest {
}
}
// @JsonIgnore
@Test
public void aggregateSecondHalf2022StorageStatusRecord() throws Exception {
public void aggregateSecondQuarter2022StorageStatusRecord() throws Exception {
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.SEPTEMBER, 1);
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.APRIL, 1);
Calendar end = Utility.getAggregationStartCalendar(2022, Calendar.JULY, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
@ -200,7 +174,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
}
}
@JsonIgnore
@Ignore
@Test
public void aggregateAllStorageStatusRecord() throws Exception {
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
@ -208,7 +182,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
// aggregateEverything(recordType);
}
// @JsonIgnore
// @Ignore
@Test
public void aggregateAllJobUsageRecord() throws Exception {
String recordType = JobUsageRecord.class.newInstance().getRecordType();
@ -216,7 +190,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
// aggregateEverything(recordType);
}
// @JsonIgnore
// @Ignore
@Test
public void aggregateAllStorageUsageRecord() throws Exception {
String recordType = StorageUsageRecord.class.newInstance().getRecordType();
@ -229,7 +203,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.DAILY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 20);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 25);
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation, false);
}
@ -272,7 +246,6 @@ public class AccountingAggregatorPluginTest extends ContextTest {
}
@JsonIgnore
@Test
public void testRecovery() throws Exception {
ContextTest.setContextByName(GCUBE);