added tests to investigate issues

pom.xml

@@ -58,7 +58,7 @@
 		<dependency>
 			<groupId>org.gcube.common</groupId>
 			<artifactId>storagehub-client-library</artifactId>
-			<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
+			<version>1.2.2</version>
 		</dependency>
 		<dependency>
 			<groupId>org.gcube.data.publishing</groupId>

src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java

@@ -184,13 +184,15 @@ public class Elaborator {
 		 * (even rare) where both check are valid because the first invocation happen 
 		 * before midnight and the second after midnight (so in the next day).
 		 */
-		if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) {
+		// if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) {
 			// Persist persist = new Persist(aggregationStatus, srcBucket, dstBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
 			Persist persist = new Persist(aggregationStatus, srcBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType);
 			persist.recover();
+		/*
 		}else{
 			logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime));
 		}
+		*/
 	}
 	
 	protected File getOriginalRecordsBackupFile(File elaborationDirectory, String name) throws Exception {

src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java

@@ -10,6 +10,7 @@ import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin.Elabora
 import org.gcube.accounting.aggregator.utility.Utility;
 import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
 import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
+import org.gcube.accounting.datamodel.usagerecords.StorageStatusRecord;
 import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -77,9 +78,64 @@ public class AccountingAggregatorPluginTest extends ContextTest {
 		
 	}
 	
+	@Test
+	public void aggregateStorageStatus() throws Exception {
+		//ContextTest.setContextByName(ROOT_DEV_SCOPE);
+		ContextTest.setContextByName(ROOT_PROD);
+		
+		Map<String, Object> inputs = new HashMap<String, Object>(); 	
+		
+		
+		AggregationType aggregationType = AggregationType.MONTHLY;
+		//type aggregation
+		inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name());
+		
+		inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name());
+		
+		
+		inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10));
+		inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59));
+		
+		
+		inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, StorageStatusRecord.class.newInstance().getRecordType());
+			
+		inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false);
+		
+		inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true);
+		inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
+		inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
+		
+		Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.MARCH, 1);
+		String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
+		logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
+		inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
+		
+		// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
+		Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2018, Calendar.JANUARY, 1);
+		/*
+		String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
+		logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
+		inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
+		*/
+		
+		AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin();
+		logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs);
+		
+		while(aggregationStartCalendar.before(aggregationEndCalendar)) {
+			
+			plugin.launch(inputs);
+			Thread.sleep(TimeUnit.SECONDS.toMillis(20));
+			plugin = new AccountingAggregatorPlugin();
+			aggregationStartCalendar.add(aggregationType.getCalendarField(), 1);
+			aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
+			inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
+		}
+		
+	}
+	
 	
 	@Test
-	public void aggregateStorage() throws Exception {
+	public void aggregateStorageUsage() throws Exception {
 		//ContextTest.setContextByName(ROOT_DEV_SCOPE);
 		ContextTest.setContextByName(ROOT_PROD);
 		
@@ -105,13 +161,13 @@ public class AccountingAggregatorPluginTest extends ContextTest {
 		inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
 		inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
 		
-		Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.AUGUST, 1);
+		Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.SEPTEMBER, 1);
 		String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
 		logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
 		inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
 		
 		// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
-		Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1);
+		Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2016, Calendar.OCTOBER, 1);
 		/*
 		String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
 		logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
@@ -161,13 +217,13 @@ public class AccountingAggregatorPluginTest extends ContextTest {
 		inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
 		inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
 		
-		Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2019, Calendar.DECEMBER, 1);
+		Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1);
 		String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
 		logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
 		inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
 		
 		// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
-		Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1);
+		Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.FEBRUARY, 1);
 		/*
 		String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
 		logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);

src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java

@@ -3,6 +3,7 @@ package org.gcube.accounting.aggregator.plugin;
 import java.text.ParseException;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -18,9 +19,16 @@ import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
 import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord;
 import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
 import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
+import org.gcube.common.storagehub.client.dsl.ContainerType;
+import org.gcube.common.storagehub.client.dsl.FolderContainer;
+import org.gcube.common.storagehub.client.dsl.ItemContainer;
+import org.gcube.common.storagehub.client.dsl.ListResolver;
+import org.gcube.common.storagehub.client.dsl.StorageHubClient;
+import org.gcube.common.storagehub.model.items.Item;
 import org.gcube.documentstore.records.AggregatedRecord;
 import org.gcube.documentstore.records.Record;
 import org.gcube.documentstore.records.RecordUtility;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -278,6 +286,31 @@ public class MyTest {
 		logger.info("{}", now.getTime().after(startTime));
 		logger.info("{}", now.getTime().before(endTime));
 		
+		
 	}
+	
+	public static final String ROOT_PROD = "/d4science.research-infrastructures.eu";
+
+	@Ignore
+	@Test
+	public void testStorageHub() throws Exception {
+		ContextTest.setContextByName(ROOT_PROD);
+		FolderContainer destinationFolder = null;
+		StorageHubClient storageHubClient = new StorageHubClient();
+		FolderContainer parent = storageHubClient.getWSRoot();
+		ListResolver listResolver = parent.findByName("Accounting-Aggregator-Plugin");
+		List<ItemContainer<? extends Item>> itemContainers = listResolver.getContainers();
+		if(itemContainers.size()>=1){
+			ItemContainer<? extends Item> itemContainer = itemContainers.get(0);
+			if(itemContainer.getType().compareTo(ContainerType.FOLDER)==0) {
+				destinationFolder = (FolderContainer) itemContainer;
+			}
+		}
+		
+		logger.debug("{}", destinationFolder);
+	}
+	
+	
+	
 
 }

src/test/java/org/gcube/accounting/aggregator/recover/LaunchGenerate.java

@@ -5,6 +5,8 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -19,7 +21,8 @@ public class LaunchGenerate {
 	@SuppressWarnings("unused")
 	private static final Logger logger = LoggerFactory.getLogger(LaunchGenerate.class);
 	
-	//@Test
+	@Ignore
+	@Test
 	public void run() throws Exception {
 		File src = new File("src");
 		File test = new File(src, "test");
@@ -40,4 +43,16 @@ public class LaunchGenerate {
 		
 	}
 	
+	@Ignore
+	@Test
+	public void testAggregationFromFile() throws Exception {
+		File src = new File("src");
+		File test = new File(src, "test");
+		File resources = new File(test, "resources");
+		File srcFile = new File(resources, "aux.json");
+		
+		TestAggregationFromFile testAggregationFromFile = new TestAggregationFromFile(srcFile);
+		testAggregationFromFile.elaborate();
+		
+	}
 }

src/test/java/org/gcube/accounting/aggregator/recover/TestAggregationFromFile.java

@@ -0,0 +1,95 @@
+package org.gcube.accounting.aggregator.recover;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.util.UUID;
+
+import org.gcube.accounting.aggregator.aggregation.AggregatorBuffer;
+import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
+import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
+import org.gcube.documentstore.exception.InvalidValueException;
+import org.gcube.documentstore.records.AggregatedRecord;
+import org.gcube.documentstore.records.Record;
+import org.gcube.documentstore.records.RecordUtility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author Luca Frosini (ISTI - CNR)
+ */
+public class TestAggregationFromFile {
+
+	protected Logger logger = LoggerFactory.getLogger(this.getClass());
+	
+	protected static final String ID = Record.ID;
+	
+	static {
+		/// One Record per package is enough
+		RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage());
+		RecordUtility.addRecordPackage(AggregatedServiceUsageRecord.class.getPackage());
+	}
+	
+	protected final File srcFile;
+	
+	protected final AggregatorBuffer aggregatorBuffer;
+	
+	protected int elaborated;
+	protected int aggregated;
+	
+	public TestAggregationFromFile(File srcFile){
+		this.srcFile = srcFile;
+		this.elaborated = 0;
+		this.aggregated = 0;
+		this.aggregatorBuffer = new AggregatorBuffer();
+	}
+	
+	protected void readFile() throws Exception {
+		try {
+			// Open the file that is the first // command line parameter
+			FileInputStream fstream = new FileInputStream(srcFile);
+			// Get the object of DataInputStream
+			DataInputStream in = new DataInputStream(fstream);
+			BufferedReader br = new BufferedReader(new InputStreamReader(in));
+			
+			String line;
+			// Read File Line By Line
+			while ((line = br.readLine()) != null) {
+				elaborateLine(line);
+				++elaborated;
+			}
+			
+			br.close();
+			in.close();
+			fstream.close();
+			
+		} catch (Exception e) {
+			logger.error("Error while elaborating file {}", srcFile.getAbsolutePath(), e);
+			throw e;
+		}
+
+	}
+
+	public void elaborate() throws Exception{
+		logger.info("Going to elaborate {}", srcFile.getAbsolutePath());
+		readFile();
+	}
+	
+	protected void elaborateLine(String json) throws Exception {
+		
+		Record record = RecordUtility.getRecord(json);
+		try {
+			record.validate();
+		}catch (InvalidValueException e) {
+			return;
+		}
+		record.setId(UUID.randomUUID().toString());
+		
+		@SuppressWarnings("rawtypes")
+		AggregatedRecord aggregatedRecord = AggregatorBuffer.getAggregatedRecord(record);
+		aggregatorBuffer.aggregate(aggregatedRecord);
+	}
+	
+}

src/test/resources/aux.json

@@ -0,0 +1,5 @@
+{"operationCount":1,"creationTime":1489770707728,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489770707728,"endTime":1489770707728,"id":"e9b6b523-c89e-4c91-a03f-4796d74ab5b1","operationResult":"SUCCESS"}
+{"operationCount":1,"creationTime":1489857068501,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489857068501,"endTime":1489857068501,"id":"28c5315f-9f70-4125-91a4-154d770f733a","operationResult":"SUCCESS"}
+{"operationCount":1,"creationTime":1489943468541,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489943468541,"endTime":1489943468541,"id":"2c8289d5-d016-4684-8045-afa2d1ca8507","operationResult":"SUCCESS"}
+{"operationCount":1,"creationTime":1490116270895,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1490116270895,"endTime":1490116270895,"id":"961a213b-8266-4d03-9dc5-8b816c0238c6","operationResult":"SUCCESS"}
+{"operationCount":1,"creationTime":1490202675117,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1490202675117,"endTime":1490202675117,"id":"0b906d52-6891-4904-98eb-7b6a4f06d9f0","operationResult":"SUCCESS"}