gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added the possibility of having partial harvesting

This commit is contained in:
Luca Frosini 2019-10-21 16:56:44 +02:00
parent d55176cc82
commit 4913a3fe03
5 changed files with 61 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pom.xml
View File

@ -12,7 +12,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-dashboard-harvester-se-plugin</artifactId>
<version>1.2.0</version>
<version>1.2.1-SNAPSHOT</version>
<name>Accounting Dashboard Harvester SmartExecutor Plugin</name>
<description>Accounting Dashboard Harvester SmartExecutor Plugin</description>
@ -51,7 +51,7 @@
<dependency>
<groupId>org.gcube.vremanagement</groupId>
<artifactId>smart-executor-api</artifactId>
<version>[1.5.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
<version>[1.5.0, 2.0.0-SNAPSHOT)</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@ -68,7 +68,7 @@
<dependency>
<groupId>org.gcube.vremanagement</groupId>
<artifactId>smart-executor-client</artifactId>
<version>[1.3.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
<version>[1.3.0,2.0.0-SNAPSHOT)</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -84,7 +84,7 @@
<dependency>
<groupId>org.gcube.portlets.admin</groupId>
<artifactId>rmp-common-library</artifactId>
<version>[2.7.2-SNAPSHOT,3.0.0-SNAPSHOT)</version>
<version>[2.7.2,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
@ -100,13 +100,13 @@
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>storagehub-client-library</artifactId>
<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.data-publishing</groupId>
<artifactId>gcat-client</artifactId>
<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
</dependency>
<!-- Dependencies forced to provided -->
@ -166,12 +166,12 @@
<dependency>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
<version>[2.0.0,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics-persistence-couchbase</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
<version>[1.0.0,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
@ -183,7 +183,7 @@
<dependency>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-summary-access</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
<version>[1.0.0,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>

19
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -50,6 +50,14 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
public static final String GET_VRE_USERS_INPUT_PARAMETER = "getVREUsers";
public static final String DRY_RUN_INPUT_PARAMETER = "dryRun";
/**
* Allows partial harvesting of data of the current period.
* This means that in MONTHLY aggregation type the current month is harvested instead of the previous month which
* is done when the month is completed.
* This allow the portlet to display monthly data in the current moth even the data is partial (till the current day).
*/
public static final String PARTIAL_HARVESTING = "partialHarvesting";
public static final String SO_BIG_DATA_VO = "/d4science.research-infrastructures.eu/SoBigData";
public static final String SO_BIG_DATA_EU_VRE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
public static final String SO_BIG_DATA_IT_VRE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it";
@ -183,14 +191,19 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
}
}
boolean partialHarvesting = false;
if(inputs.containsKey(PARTIAL_HARVESTING)) {
partialHarvesting = (boolean) inputs.get(PARTIAL_HARVESTING);
}
if(inputs.containsKey(START_DATE_INPUT_PARAMETER)) {
String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER);
start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC);
} else {
start = DateUtils.getPreviousPeriod(aggregationType).getTime();
start = DateUtils.getPreviousPeriod(aggregationType, partialHarvesting).getTime();
}
end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, partialHarvesting);
logger.debug("Harvesting from {} to {} (ReRun:{} - GetVREUsers:{} - DryRun:{})",
DateUtils.format(start), DateUtils.format(end), reRun, getVREUsers, dryRun);
@ -310,7 +323,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
if(getVREUsers) {
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
// The VREUsers can be only Harvested for the last month
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType, partialHarvesting).getTime())) {
logger.info("Going to harvest Context Users for {}", context);
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);

24
src/main/java/org/gcube/dataharvest/utils/DateUtils.java
View File

@ -44,23 +44,29 @@ public class DateUtils {
return Calendar.getInstance(UTC_TIMEZONE);
}
public static Calendar getPreviousPeriod(AggregationType aggregationType) {
public static Calendar getPreviousPeriod(AggregationType aggregationType, boolean partialHarvesting) {
Calendar now = getUTCCalendarInstance();
switch(aggregationType) {
case YEARLY:
now.add(Calendar.YEAR, -1);
if(!partialHarvesting) {
now.add(Calendar.YEAR, -1);
}
now.set(Calendar.MONTH, Calendar.JANUARY);
now.set(Calendar.DAY_OF_MONTH, 1);
break;
case MONTHLY:
now.add(Calendar.MONTH, -1);
if(!partialHarvesting) {
now.add(Calendar.MONTH, -1);
}
now.set(Calendar.DAY_OF_MONTH, 1);
break;
case DAILY:
now.add(Calendar.DAY_OF_MONTH, -1);
if(!partialHarvesting) {
now.add(Calendar.DAY_OF_MONTH, -1);
}
break;
default:
@ -91,11 +97,13 @@ public class DateUtils {
return aggregationStartCalendar;
}
public static Date getEndDateFromStartDate(AggregationType aggregationType, Date startDate, int offset) {
public static Date getEndDateFromStartDate(AggregationType aggregationType, Date startDate, int offset, boolean partialHarvesting) {
Calendar aggregationEndDate = getUTCCalendarInstance();
aggregationEndDate.setTimeInMillis(startDate.getTime());
aggregationEndDate.add(aggregationType.getCalendarField(), offset);
aggregationEndDate.add(Calendar.MILLISECOND, -1);
if(!partialHarvesting) {
aggregationEndDate.setTimeInMillis(startDate.getTime());
aggregationEndDate.add(aggregationType.getCalendarField(), offset);
aggregationEndDate.add(Calendar.MILLISECOND, -1);
}
return aggregationEndDate.getTime();
}

39
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -105,7 +105,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false);
inputs.put(AccountingDataHarvesterPlugin.PARTIAL_HARVESTING, true);
/*
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
@ -227,7 +228,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
while(from.before(runbeforeDate)) {
Date start = from.getTime();
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false);
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
@ -311,7 +312,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime();
// start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false);
logger.info("\n\n\n");
@ -321,7 +322,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
try {
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType, false).getTime())) {
logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
} else {
@ -370,7 +371,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
List<AccountingRecord> accountingRecords = new ArrayList<>();
for(Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
ContextTest.setContextByName(ROOT);
VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end);
@ -409,8 +410,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
accountingDataHarvesterPlugin.getConfigParameters();
@ -501,7 +502,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Calendar from = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
Date start = from.getTime();
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false);
logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end));
@ -546,8 +547,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
@ -616,7 +617,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
for(Date start : starts) {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start,
end);
@ -653,8 +654,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
List<AccountingRecord> harvested = vreUsersHarvester.getAccountingRecords();
@ -678,8 +679,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AggregationType measureType = AggregationType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
SortedSet<String> contexts = getContexts();
@ -709,8 +710,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
accountingDataHarvesterPlugin.getConfigParameters();
@ -740,8 +741,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
accountingDataHarvesterPlugin.getConfigParameters();

2
target/.gitignore vendored
View File

@ -1,2 +0,0 @@
/classes/
/test-classes/