gCubeSystem
/
accounting-aggregator-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixing plugin

This commit is contained in:
Luca Frosini 2024-02-16 17:06:49 +01:00
parent 13282e6236
commit 210361a0ff
8 changed files with 139 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pom.xml
View File

@ -57,15 +57,15 @@
<artifactId>document-store-lib-postgresql</artifactId>
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics-persistence-postgresql</artifactId>
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.gcube.accounting</groupId>-->
<!-- <artifactId>accounting-analytics-persistence-postgresql</artifactId>-->
<!-- <version>[2.0.1-SNAPSHOT, 3.0.0-SNAPSHOT)</version>-->
<!-- </dependency>-->
<dependency>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics</artifactId>
<version>[3.0.0, 4.0.0-SNAPSHOT)</version>
<version>[4.0.0, 5.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@ -135,4 +135,4 @@
</plugin>
</plugins>
</build>
</project>
</project>

1
src/main/java/org/gcube/accounting/aggregator/aggregation/Aggregator.java
View File

@ -219,7 +219,6 @@ public class Aggregator {
Class<? extends AggregatedRecord<?, ?>> clz = RecordUtility.getAggregatedRecordClass(type);
RecordToDBFields recordToDBFields = RecordToDBMapping.getRecordToDBFields(clz);
Set<String> requiredFields = clz.newInstance().getRequiredFields();
malformedRecordNumber = 0;

22
src/main/java/org/gcube/accounting/aggregator/persist/DocumentElaboration.java
View File

@ -24,9 +24,6 @@ public abstract class DocumentElaboration {
protected static final String ID = Record.ID;
protected static final int THRESHOLD_FOR_FIVE_PERCENT = 10000;
protected static final int THRESHOLD_FOR_ONE_PERCENT = 100000;
public static final int MAX_RETRY = 7;
protected final AggregationStatus aggregationStatus;
@ -59,13 +56,9 @@ public abstract class DocumentElaboration {
logger.info("{} - Going to elaborate {} rows", aggregationStatus.getAggregationInfo(), rowToBeElaborated);
int percentOfNumberOfRows = (rowToBeElaborated / 10) + 1;
if(rowToBeElaborated >= THRESHOLD_FOR_FIVE_PERCENT) {
percentOfNumberOfRows = percentOfNumberOfRows / 2;
}
if(rowToBeElaborated >= THRESHOLD_FOR_ONE_PERCENT) {
percentOfNumberOfRows = percentOfNumberOfRows / 5;
int numberOfRowsForEachRecoveryPoint = (rowToBeElaborated / 10) + 1;
if(numberOfRowsForEachRecoveryPoint>500) {
numberOfRowsForEachRecoveryPoint = 500;
}
currentlyElaborated = 0;
@ -109,7 +102,7 @@ public abstract class DocumentElaboration {
}
++currentlyElaborated;
if(currentlyElaborated % percentOfNumberOfRows == 0) {
if(currentlyElaborated % numberOfRowsForEachRecoveryPoint == 0) {
if(currentlyElaborated>=restartFrom) {
aggregationStatus.setRestartFrom(currentlyElaborated, true);
}
@ -152,11 +145,12 @@ public abstract class DocumentElaboration {
startTime = Utility.getUTCCalendarInstance();
try {
readFile();
aggregationStatus.setAggregationState(finalAggregationState, startTime, true);
}catch (Exception e) {
throw e;
} finally {
afterElaboration();
}
afterElaboration();
aggregationStatus.setAggregationState(finalAggregationState, startTime, true);
}
protected abstract void elaborateLine(String line) throws Exception;

8
src/main/java/org/gcube/accounting/aggregator/persist/InsertDocument.java
View File

@ -113,10 +113,10 @@ public class InsertDocument extends DocumentElaboration {
aggregatorPersistenceDst.insert(record);
++count;
if(count==100) {
aggregatorPersistenceDst.commitAndClose();
count = 0;
}
// if(count==100) {
// aggregatorPersistenceDst.commitAndClose();
// count = 0;
// }
}

14
src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnector.java
View File

@ -12,7 +12,6 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.UUID;
@ -29,10 +28,8 @@ import org.gcube.accounting.utility.postgresql.RecordToDBFields;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.documentstore.persistence.PersistencePostgreSQL;
import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.DSMapper;
import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.records.RecordUtility;
import org.postgresql.core.Utils;
/**
@ -49,16 +46,9 @@ public class PostgreSQLConnector extends PersistencePostgreSQL implements Aggreg
protected Connection connection;
protected PostgreSQLConnector(Class<?> clazz) throws Exception {
super();
this.configuration = new AccountingPersistenceConfiguration(clazz);
Map<String, Class<? extends AggregatedRecord<?,?>>> aggregatedRecords = RecordUtility.getAggregatedRecordClassesFound();
for(String typeName : aggregatedRecords.keySet()) {
try {
Class<? extends AggregatedRecord<?,?>> clz = aggregatedRecords.get(typeName);
RecordToDBMapping.addRecordToDB(clz, configuration);
} catch (Exception e) {
new RuntimeException(e);
}
}
prepareConnection(configuration);
}
protected Connection getConnection() throws Exception {

4
src/main/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorStatus.java
View File

@ -1,14 +1,12 @@
package org.gcube.accounting.aggregator.persistence;
import org.gcube.accounting.analytics.persistence.postgresql.AccountingPersistenceQueryPostgreSQL;
/**
* @author Luca Frosini (ISTI-CNR)
*/
public class PostgreSQLConnectorStatus extends PostgreSQLConnector implements AggregatorPersistenceStatus {
protected PostgreSQLConnectorStatus() throws Exception {
super(AccountingPersistenceQueryPostgreSQL.class);
super(AggregatorPersistenceSrc.class);
}
protected PostgreSQLConnectorStatus(Class<?> clazz) throws Exception {

5
src/test/java/org/gcube/accounting/aggregator/persistence/PostgreSQLConnectorTest.java
View File

@ -26,16 +26,17 @@ public class PostgreSQLConnectorTest extends ContextTest {
public PostgreSQLConnectorTest() throws Exception {
aggregatorPersistenceStatus = AggregatorPersistenceFactory.getAggregatorPersistenceStatus();
logger.debug("{}", aggregatorPersistenceStatus);
}
@Ignore
// @Ignore
@Test
public void getAggregatorPersistenceDst() throws Exception {
AggregatorPersistenceDst dst = AggregatorPersistenceFactory.getAggregatorPersistenceDst();
logger.debug("{}", dst);
}
@Ignore
// @Ignore
@Test
public void getAggregatorPersistenceSrc() throws Exception {
AggregatorPersistenceSrc src = AggregatorPersistenceFactory.getAggregatorPersistenceSrc();

130
src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java
View File

@ -74,40 +74,139 @@ public class AccountingAggregatorPluginTest extends ContextTest {
}
}
@JsonIgnore
@Test
public void aggregateAllServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
// aggregateOneShot(recordType);
aggregateEverything(recordType);
}
// @JsonIgnore
@Test
public void aggregateDailyServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.DAILY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 6);
Calendar end = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 14);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation);
aggregationStartCalendar = Calendar.getInstance();
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
}
}
// @JsonIgnore
@Test
public void aggregateMonthlyFirstHalf2021ServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.FEBRUARY, 1);
Calendar end = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation);
aggregationStartCalendar = Calendar.getInstance();
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
}
}
// @JsonIgnore
@Test
public void aggregateMonthlySecondHalf2021ServiceUsageRecord() throws Exception {
String recordType = ServiceUsageRecord.class.newInstance().getRecordType();
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.AUGUST, 1);
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation);
aggregationStartCalendar = Calendar.getInstance();
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
}
}
@JsonIgnore
@Test
public void aggregateAllStorageUsageRecord() throws Exception {
String recordType = StorageUsageRecord.class.newInstance().getRecordType();
aggregateEverything(recordType);
public void aggregateAllStorageStatusRecord() throws Exception {
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
// aggregateOneShot(recordType);
// aggregateEverything(recordType);
}
// @JsonIgnore
@Test
public void aggregateMonthlyStorageStatusRecord() throws Exception {
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1);
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.NOVEMBER, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation);
aggregationStartCalendar = Calendar.getInstance();
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
}
}
// @JsonIgnore
@Test
public void aggregateSecondHalf2022StorageStatusRecord() throws Exception {
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
boolean forceRestart = true;
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.MONTHLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2022, Calendar.JUNE, 1);
Calendar end = Utility.getAggregationStartCalendar(2023, Calendar.JANUARY, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation);
aggregationStartCalendar = Calendar.getInstance();
aggregationStartCalendar.setTimeInMillis(aggregationEndCalendar.getTimeInMillis());
}
}
@JsonIgnore
@Test
public void aggregateAllJobUsageRecord() throws Exception {
String recordType = JobUsageRecord.class.newInstance().getRecordType();
aggregateEverything(recordType);
}
@Test
public void aggregateAllStorageStatusRecord() throws Exception {
String recordType = StorageStatusRecord.class.newInstance().getRecordType();
aggregateOneShot(recordType);
// aggregateEverything(recordType);
// aggregateEverything(recordType);
}
@JsonIgnore
@Test
public void aggregateAllStorageUsageRecord() throws Exception {
String recordType = StorageUsageRecord.class.newInstance().getRecordType();
aggregateOneShot(recordType);
// aggregateEverything(recordType);
}
public void aggregateOneShot(String recordType) throws Exception {
boolean forceRestart = true;
boolean forceEarlyAggregation = false;
AggregationType aggregationType = AggregationType.YEARLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1);
boolean forceEarlyAggregation = true;
AggregationType aggregationType = AggregationType.DAILY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2024, Calendar.FEBRUARY, 14);
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);
aggregate(recordType, aggregationType, aggregationStartCalendar, aggregationEndCalendar, forceRestart, forceEarlyAggregation);
}
@ -117,9 +216,8 @@ public class AccountingAggregatorPluginTest extends ContextTest {
boolean forceRestart = true;
boolean forceEarlyAggregation = false;
AggregationType aggregationType = AggregationType.YEARLY;
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2015, Calendar.JANUARY, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.JANUARY, 1);
Calendar end = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1);
while (aggregationStartCalendar.before(end)) {
Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(aggregationType, aggregationStartCalendar, 1);