Merge branch 'feature/20952' into feature/21347
Conflicts: src/test/java/org/gcube/accounting/aggregator/plugin/CouchBaseConnectorTest.java
This commit is contained in:
commit
26a8d2649a
9
pom.xml
9
pom.xml
|
@ -32,7 +32,7 @@
|
|||
<dependency>
|
||||
<groupId>org.gcube.vremanagement</groupId>
|
||||
<artifactId>smart-executor-bom</artifactId>
|
||||
<version>2.0.0-SNAPSHOT</version>
|
||||
<version>3.1.0-SNAPSHOT</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
|
@ -63,6 +63,11 @@
|
|||
<artifactId>accounting-analytics-persistence-postgresql</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.accounting</groupId>
|
||||
<artifactId>accounting-analytics</artifactId>
|
||||
<version>[3.0.0, 4.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
|
@ -80,7 +85,6 @@
|
|||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
|
@ -104,5 +108,4 @@
|
|||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
|
@ -10,6 +10,7 @@ import org.gcube.accounting.aggregator.aggregation.AggregationType;
|
|||
import org.gcube.accounting.aggregator.aggregation.Aggregator;
|
||||
import org.gcube.accounting.aggregator.directory.FileSystemDirectoryStructure;
|
||||
import org.gcube.accounting.aggregator.persist.Persist;
|
||||
import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin;
|
||||
import org.gcube.accounting.aggregator.status.AggregationState;
|
||||
import org.gcube.accounting.aggregator.status.AggregationStatus;
|
||||
import org.gcube.accounting.aggregator.utility.Constant;
|
||||
|
@ -178,15 +179,14 @@ public class Elaborator {
|
|||
* (even rare) where both check are valid because the first invocation happen
|
||||
* before midnight and the second after midnight (so in the next day).
|
||||
*/
|
||||
// if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) {
|
||||
if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) {
|
||||
// Persist persist = new Persist(aggregationStatus, srcBucket, dstBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
|
||||
Persist persist = new Persist(aggregationStatus, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
|
||||
persist.recover();
|
||||
/*
|
||||
}else{
|
||||
logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime));
|
||||
}
|
||||
*/
|
||||
|
||||
}
|
||||
|
||||
protected File getOriginalRecordsBackupFile(File elaborationDirectory, String name) throws Exception {
|
||||
|
|
|
@ -105,13 +105,13 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
|||
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
|
||||
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
|
||||
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2018, Calendar.APRIL, 1);
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.MAY, 1);
|
||||
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
|
||||
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1);
|
||||
/*
|
||||
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
|
||||
|
@ -161,13 +161,13 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
|||
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
|
||||
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
|
||||
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.APRIL, 1);
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.APRIL, 1);
|
||||
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
|
||||
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.MAY, 1);
|
||||
/*
|
||||
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
|
||||
|
@ -217,13 +217,13 @@ public class AccountingAggregatorPluginTest extends ContextTest {
|
|||
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
|
||||
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
|
||||
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1);
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JUNE, 1);
|
||||
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
|
||||
|
||||
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JANUARY, 1);
|
||||
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2021, Calendar.JULY, 1);
|
||||
/*
|
||||
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
|
||||
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
|
||||
|
|
|
@ -1,176 +0,0 @@
|
|||
package org.gcube.accounting.aggregator.plugin;
|
||||
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.accounting.aggregator.aggregation.AggregationInfo;
|
||||
import org.gcube.accounting.aggregator.aggregation.AggregationType;
|
||||
import org.gcube.accounting.aggregator.persistence.PostgreSQLConnector;
|
||||
import org.gcube.accounting.aggregator.status.AggregationState;
|
||||
import org.gcube.accounting.aggregator.status.AggregationStatus;
|
||||
import org.gcube.accounting.aggregator.utility.Utility;
|
||||
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||
import org.gcube.documentstore.records.DSMapper;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class CouchBaseConnectorTest extends ContextTest {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(AccountingAggregatorPluginTest.class);
|
||||
|
||||
private PostgreSQLConnector postgreSQLConnector;
|
||||
|
||||
public CouchBaseConnectorTest() throws Exception {
|
||||
ContextTest.setContextByName(ROOT_DEV_SCOPE);
|
||||
postgreSQLConnector = new PostgreSQLConnector();
|
||||
}
|
||||
|
||||
protected void analyseAggregationStatus(AggregationStatus aggregationStatus) throws Exception {
|
||||
if(aggregationStatus.getPrevious()!=null) {
|
||||
analyseAggregationStatus(aggregationStatus.getPrevious());
|
||||
}
|
||||
if(aggregationStatus.getOriginalRecordsNumber()!=0) {
|
||||
postgreSQLConnector.upsertAggregationStatus(aggregationStatus);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void insertAllInPostgreSQL() throws Exception {
|
||||
ContextTest.setContextByName(ROOT_DEV_SCOPE);
|
||||
List<AggregationStatus> aggregationStatuses = postgreSQLConnector.getAll();
|
||||
for(AggregationStatus aggregationStatus : aggregationStatuses) {
|
||||
analyseAggregationStatus(aggregationStatus);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getLastTest() throws Exception {
|
||||
AggregationStatus aggregationStatus = postgreSQLConnector.getLast(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, null, null);
|
||||
logger.debug("Last : {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getUnterminatedTest() throws Exception{
|
||||
List<AggregationStatus> aggregationStatuses = postgreSQLConnector.getUnterminated(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, null, null);
|
||||
for(AggregationStatus aggregationStatus : aggregationStatuses){
|
||||
logger.debug("Unterminated : {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getLastTestWithConstraint() throws Exception {
|
||||
Calendar aggregationStart = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1);
|
||||
Calendar aggregationEnd = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 31);
|
||||
|
||||
AggregationStatus aggregationStatus = postgreSQLConnector.getLast(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, aggregationStart.getTime(), aggregationEnd.getTime());
|
||||
logger.info("Last : {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getUnterminatedTestWithConstraint() throws Exception{
|
||||
Calendar aggregationStart = Utility.getAggregationStartCalendar(2017, Calendar.APRIL, 1);
|
||||
Calendar aggregationEnd = Utility.getAggregationStartCalendar(2017, Calendar.APRIL, 30);
|
||||
|
||||
List<AggregationStatus> aggregationStatuses = postgreSQLConnector.getUnterminated(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, aggregationStart.getTime(), aggregationEnd.getTime());
|
||||
for(AggregationStatus aggregationStatus : aggregationStatuses){
|
||||
logger.info("Unterminated : {}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getAggregationStatusTest() throws Exception{
|
||||
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 15);
|
||||
AggregationStatus aggregationStatus = postgreSQLConnector.getAggregationStatus(ServiceUsageRecord.class.getSimpleName(), AggregationType.DAILY, aggregationStartCalendar.getTime());
|
||||
logger.debug("{}", DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void aggregationStatusTest() throws Exception {
|
||||
int toRemove = -36;
|
||||
|
||||
Calendar today = Utility.getUTCCalendarInstance();
|
||||
today.add(Calendar.DAY_OF_YEAR, toRemove);
|
||||
|
||||
String aggregationStartDateString = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(today.getTime());
|
||||
Date aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.parse(aggregationStartDateString);
|
||||
|
||||
Calendar tomorrow = Utility.getUTCCalendarInstance();
|
||||
tomorrow.add(Calendar.DAY_OF_YEAR, toRemove+1);
|
||||
String aggregationEndDateString = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(tomorrow.getTime());
|
||||
Date aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.parse(aggregationEndDateString);
|
||||
|
||||
|
||||
AggregationInfo aggregation = new AggregationInfo(ServiceUsageRecord.class.newInstance().getRecordType(), AggregationType.DAILY, aggregationStartDate, aggregationEndDate);
|
||||
String aggregationString = DSMapper.getObjectMapper().writeValueAsString(aggregation);
|
||||
logger.debug("{} : {}", AggregationInfo.class.getSimpleName(), aggregationString);
|
||||
|
||||
AggregationStatus aggregationStatus = new AggregationStatus(aggregation);
|
||||
aggregationStatus.setContext("TEST_CONTEXT");
|
||||
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
// Set to true just for one test and restore to false
|
||||
boolean sync = true;
|
||||
|
||||
Calendar startedStart = Utility.getUTCCalendarInstance();
|
||||
aggregationStatus.setAggregationState(AggregationState.STARTED, startedStart, sync);
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
aggregationStatus.setRecordNumbers(100, 72, 0);
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
Calendar aggregatedStart = Utility.getUTCCalendarInstance();
|
||||
aggregationStatus.setAggregationState(AggregationState.AGGREGATED, aggregatedStart, sync);
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
Calendar addedStart = Utility.getUTCCalendarInstance();
|
||||
aggregationStatus.setAggregationState(AggregationState.ADDED, addedStart, sync);
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
Calendar deletedStart = Utility.getUTCCalendarInstance();
|
||||
aggregationStatus.setAggregationState(AggregationState.DELETED, deletedStart, sync);
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
Calendar completedStart = Utility.getUTCCalendarInstance();
|
||||
aggregationStatus.setAggregationState(AggregationState.COMPLETED, completedStart, sync);
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void createStartedElaboration() throws Exception {
|
||||
|
||||
Calendar start = Utility.getAggregationStartCalendar(2017, Calendar.JUNE, 15);
|
||||
String aggregationStartDateString = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(start.getTime());
|
||||
Date aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.parse(aggregationStartDateString);
|
||||
|
||||
Calendar end = Utility.getUTCCalendarInstance();
|
||||
end.setTime(aggregationStartDate);
|
||||
end.add(Calendar.DAY_OF_MONTH, 1);
|
||||
String aggregationEndDateString = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(end.getTime());
|
||||
Date aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.parse(aggregationEndDateString);
|
||||
|
||||
|
||||
AggregationInfo aggregation = new AggregationInfo(ServiceUsageRecord.class.newInstance().getRecordType(), AggregationType.DAILY, aggregationStartDate, aggregationEndDate);
|
||||
String aggregationString = DSMapper.getObjectMapper().writeValueAsString(aggregation);
|
||||
logger.debug("{} : {}", AggregationInfo.class.getSimpleName(), aggregationString);
|
||||
|
||||
AggregationStatus aggregationStatus = new AggregationStatus(aggregation);
|
||||
aggregationStatus.setContext("TEST_CONTEXT");
|
||||
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
// Set to true just for one test and restore to false
|
||||
boolean sync = true;
|
||||
|
||||
Calendar startedStart = Utility.getUTCCalendarInstance();
|
||||
aggregationStatus.setAggregationState(AggregationState.STARTED, startedStart, sync);
|
||||
logger.debug("{} : {}", AggregationStatus.class.getSimpleName(), DSMapper.getObjectMapper().writeValueAsString(aggregationStatus));
|
||||
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue