From 4913a3fe032c201304566ad2f2fe635b2763ec24 Mon Sep 17 00:00:00 2001
From: Luca Frosini <luca.frosini@isti.cnr.it>
Date: Mon, 21 Oct 2019 16:56:44 +0200
Subject: [PATCH] Added the possibility of having partial harvesting

---
 pom.xml                                       | 18 ++++-----
 .../AccountingDataHarvesterPlugin.java        | 19 +++++++--
 .../gcube/dataharvest/utils/DateUtils.java    | 24 ++++++++----
 .../AccountingDataHarvesterPluginTest.java    | 39 ++++++++++---------
 target/.gitignore                             |  2 -
 5 files changed, 61 insertions(+), 41 deletions(-)
 delete mode 100644 target/.gitignore

diff --git a/pom.xml b/pom.xml
index c9ccdee..7701f2e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -12,7 +12,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.gcube.accounting</groupId>
 	<artifactId>accounting-dashboard-harvester-se-plugin</artifactId>
-	<version>1.2.0</version>
+	<version>1.2.1-SNAPSHOT</version>
 	<name>Accounting Dashboard Harvester SmartExecutor Plugin</name>
 	<description>Accounting Dashboard Harvester SmartExecutor Plugin</description>
 
@@ -51,7 +51,7 @@
 		<dependency>
 			<groupId>org.gcube.vremanagement</groupId>
 			<artifactId>smart-executor-api</artifactId>
-			<version>[1.5.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
+			<version>[1.5.0, 2.0.0-SNAPSHOT)</version>
 			<scope>provided</scope>
 			<exclusions>
 				<exclusion>
@@ -68,7 +68,7 @@
 		<dependency>
 			<groupId>org.gcube.vremanagement</groupId>
 			<artifactId>smart-executor-client</artifactId>
-			<version>[1.3.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
+			<version>[1.3.0,2.0.0-SNAPSHOT)</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
@@ -84,7 +84,7 @@
 		<dependency>
 			<groupId>org.gcube.portlets.admin</groupId>
 			<artifactId>rmp-common-library</artifactId>
-			<version>[2.7.2-SNAPSHOT,3.0.0-SNAPSHOT)</version>
+			<version>[2.7.2,3.0.0-SNAPSHOT)</version>
 		</dependency>
 		<dependency>
 			<groupId>com.google.apis</groupId>
@@ -100,13 +100,13 @@
 		<dependency>
 			<groupId>org.gcube.common</groupId>
 			<artifactId>storagehub-client-library</artifactId>
-			<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
+			<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.gcube.data-publishing</groupId>
 			<artifactId>gcat-client</artifactId>
-			<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
+			<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
 		</dependency>
 
 		<!-- Dependencies forced to provided -->
@@ -166,12 +166,12 @@
 		<dependency>
 			<groupId>org.gcube.accounting</groupId>
 			<artifactId>accounting-analytics</artifactId>
-			<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
+			<version>[2.0.0,3.0.0-SNAPSHOT)</version>
 		</dependency>
 		<dependency>
 			<groupId>org.gcube.accounting</groupId>
 			<artifactId>accounting-analytics-persistence-couchbase</artifactId>
-			<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
+			<version>[1.0.0,2.0.0-SNAPSHOT)</version>
 		</dependency>
 
 		<dependency>
@@ -183,7 +183,7 @@
 		<dependency>
 			<groupId>org.gcube.accounting</groupId>
 			<artifactId>accounting-summary-access</artifactId>
-			<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
+			<version>[1.0.0,2.0.0-SNAPSHOT)</version>
 		</dependency>
 
 		<dependency>
diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
index 68ecb46..de9bc3d 100644
--- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
+++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
@@ -50,6 +50,14 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 	public static final String GET_VRE_USERS_INPUT_PARAMETER = "getVREUsers";
 	public static final String DRY_RUN_INPUT_PARAMETER = "dryRun";
 	
+	/**
+	 * Allows partial harvesting of data of the current period.
+	 * This means that in MONTHLY aggregation type the current month is harvested instead of the previous month which
+	 * is done when the month is completed.
+	 * This allow the portlet to display monthly data in the current moth even the data is partial (till the current day).
+	 */
+	public static final String PARTIAL_HARVESTING = "partialHarvesting";
+	
 	public static final String SO_BIG_DATA_VO = "/d4science.research-infrastructures.eu/SoBigData";
 	public static final String SO_BIG_DATA_EU_VRE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
 	public static final String SO_BIG_DATA_IT_VRE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.it";
@@ -183,14 +191,19 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 			}
 		}
 		
+		boolean partialHarvesting = false;
+		if(inputs.containsKey(PARTIAL_HARVESTING)) {
+			partialHarvesting = (boolean) inputs.get(PARTIAL_HARVESTING);
+		}
+		
 		if(inputs.containsKey(START_DATE_INPUT_PARAMETER)) {
 			String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER);
 			start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC);
 		} else {
-			start = DateUtils.getPreviousPeriod(aggregationType).getTime();
+			start = DateUtils.getPreviousPeriod(aggregationType, partialHarvesting).getTime();
 		}
 		
-		end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
+		end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, partialHarvesting);
 		
 		logger.debug("Harvesting from {} to {} (ReRun:{} - GetVREUsers:{} - DryRun:{})", 
 				DateUtils.format(start), DateUtils.format(end), reRun, getVREUsers, dryRun);
@@ -310,7 +323,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 					if(getVREUsers) {
 						// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
 						// The VREUsers can be only Harvested for the last month
-						if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
+						if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType, partialHarvesting).getTime())) {
 							logger.info("Going to harvest Context Users for {}", context);
 							VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
 							
diff --git a/src/main/java/org/gcube/dataharvest/utils/DateUtils.java b/src/main/java/org/gcube/dataharvest/utils/DateUtils.java
index 4e472c8..edb0660 100644
--- a/src/main/java/org/gcube/dataharvest/utils/DateUtils.java
+++ b/src/main/java/org/gcube/dataharvest/utils/DateUtils.java
@@ -44,23 +44,29 @@ public class DateUtils {
 		return Calendar.getInstance(UTC_TIMEZONE);
 	}
 	
-	public static Calendar getPreviousPeriod(AggregationType aggregationType) {
+	public static Calendar getPreviousPeriod(AggregationType aggregationType, boolean partialHarvesting) {
 		Calendar now = getUTCCalendarInstance();
 		switch(aggregationType) {
 			
 			case YEARLY:
-				now.add(Calendar.YEAR, -1);
+				if(!partialHarvesting) {
+					now.add(Calendar.YEAR, -1);
+				}
 				now.set(Calendar.MONTH, Calendar.JANUARY);
 				now.set(Calendar.DAY_OF_MONTH, 1);
 				break;
 			
 			case MONTHLY:
-				now.add(Calendar.MONTH, -1);
+				if(!partialHarvesting) {
+					now.add(Calendar.MONTH, -1);
+				}
 				now.set(Calendar.DAY_OF_MONTH, 1);
 				break;
 			
 			case DAILY:
-				now.add(Calendar.DAY_OF_MONTH, -1);
+				if(!partialHarvesting) {
+					now.add(Calendar.DAY_OF_MONTH, -1);
+				}
 				break;
 			
 			default:
@@ -91,11 +97,13 @@ public class DateUtils {
 		return aggregationStartCalendar;
 	}
 	
-	public static Date getEndDateFromStartDate(AggregationType aggregationType, Date startDate, int offset) {
+	public static Date getEndDateFromStartDate(AggregationType aggregationType, Date startDate, int offset, boolean partialHarvesting) {
 		Calendar aggregationEndDate = getUTCCalendarInstance();
-		aggregationEndDate.setTimeInMillis(startDate.getTime());
-		aggregationEndDate.add(aggregationType.getCalendarField(), offset);
-		aggregationEndDate.add(Calendar.MILLISECOND, -1);
+		if(!partialHarvesting) {
+			aggregationEndDate.setTimeInMillis(startDate.getTime());
+			aggregationEndDate.add(aggregationType.getCalendarField(), offset);
+			aggregationEndDate.add(Calendar.MILLISECOND, -1);
+		}
 		return aggregationEndDate.getTime();
 	}
 	
diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
index 540d434..c5f51bc 100644
--- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
+++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
@@ -105,7 +105,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
 			inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true);
 			inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
-			inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
+			inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false);
+			inputs.put(AccountingDataHarvesterPlugin.PARTIAL_HARVESTING, true);
 			
 			/*
 			Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
@@ -227,7 +228,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			
 			while(from.before(runbeforeDate)) {
 				Date start = from.getTime();
-				Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
+				Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false);
 				
 				logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
 				
@@ -311,7 +312,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 		
 		Date start = DateUtils.getStartCalendar(2018, Calendar.MARCH, 1).getTime();
 		// start = DateUtils.getPreviousPeriod(measureType).getTime();
-		Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
+		Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false);
 		
 		logger.info("\n\n\n");
 		
@@ -321,7 +322,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			
 			try {
 				
-				if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
+				if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType, false).getTime())) {
 					logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
 							DateUtils.format(end));
 				} else {
@@ -370,7 +371,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			List<AccountingRecord> accountingRecords = new ArrayList<>();
 			
 			for(Date start : starts) {
-				Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+				Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
 				
 				ContextTest.setContextByName(ROOT);
 				VREAccessesHarvester vreAccessesHarvester = new VREAccessesHarvester(start, end);
@@ -409,8 +410,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
 			// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
 			
-			Date start = DateUtils.getPreviousPeriod(measureType).getTime();
-			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
+			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
 			
 			AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
 			accountingDataHarvesterPlugin.getConfigParameters();
@@ -501,7 +502,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			
 			Calendar from = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
 			Date start = from.getTime();
-			Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
+			Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1, false);
 			
 			logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end));
 			
@@ -546,8 +547,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			
 			AggregationType measureType = AggregationType.MONTHLY;
 			
-			Date start = DateUtils.getPreviousPeriod(measureType).getTime();
-			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
+			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
 			
 			MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
 			List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
@@ -616,7 +617,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			
 			for(Date start : starts) {
 				
-				Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+				Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
 				
 				TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start,
 						end);
@@ -653,8 +654,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			
 			AggregationType measureType = AggregationType.MONTHLY;
 			
-			Date start = DateUtils.getPreviousPeriod(measureType).getTime();
-			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
+			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
 			
 			VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
 			List<AccountingRecord> harvested = vreUsersHarvester.getAccountingRecords();
@@ -678,8 +679,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			
 			AggregationType measureType = AggregationType.MONTHLY;
 			
-			Date start = DateUtils.getPreviousPeriod(measureType).getTime();
-			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
+			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
 			
 			SortedSet<String> contexts = getContexts();
 			
@@ -709,8 +710,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
 			// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
 			
-			Date start = DateUtils.getPreviousPeriod(measureType).getTime();
-			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
+			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
 			
 			AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
 			accountingDataHarvesterPlugin.getConfigParameters();
@@ -740,8 +741,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			// Date start = DateUtils.getStartCalendar(2015, Calendar.FEBRUARY, 1).getTime();
 			// Date end = DateUtils.getStartCalendar(2019, Calendar.FEBRUARY, 1).getTime();
 			
-			Date start = DateUtils.getPreviousPeriod(measureType).getTime();
-			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
+			Date start = DateUtils.getPreviousPeriod(measureType, false).getTime();
+			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1, false);
 			
 			AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
 			accountingDataHarvesterPlugin.getConfigParameters();
diff --git a/target/.gitignore b/target/.gitignore
deleted file mode 100644
index 4569837..0000000
--- a/target/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/classes/
-/test-classes/