From bc4d09aeafcf7f302e2525a9f7e8ef02af23edb2 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Thu, 15 Apr 2021 14:58:12 +0200 Subject: [PATCH] added tests to investigate issues --- pom.xml | 2 +- .../aggregator/elaboration/Elaborator.java | 4 +- .../AccountingAggregatorPluginTest.java | 66 ++++++++++++- .../accounting/aggregator/plugin/MyTest.java | 33 +++++++ .../aggregator/recover/LaunchGenerate.java | 17 +++- .../recover/TestAggregationFromFile.java | 95 +++++++++++++++++++ src/test/resources/aux.json | 5 + 7 files changed, 214 insertions(+), 8 deletions(-) create mode 100644 src/test/java/org/gcube/accounting/aggregator/recover/TestAggregationFromFile.java create mode 100644 src/test/resources/aux.json diff --git a/pom.xml b/pom.xml index cb7158f..0f3f12f 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ org.gcube.common storagehub-client-library - [1.0.0, 2.0.0-SNAPSHOT) + 1.2.2 org.gcube.data.publishing diff --git a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java index 356222f..a42d95e 100644 --- a/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java +++ b/src/main/java/org/gcube/accounting/aggregator/elaboration/Elaborator.java @@ -184,13 +184,15 @@ public class Elaborator { * (even rare) where both check are valid because the first invocation happen * before midnight and the second after midnight (so in the next day). */ - if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) { + // if (Utility.isTimeElapsed(now, persistStartTime) && !Utility.isTimeElapsed(now, persistEndTime)) { // Persist persist = new Persist(aggregationStatus, srcBucket, dstBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); Persist persist = new Persist(aggregationStatus, srcBucket, originalRecordsbackupFile, aggregateRecordsBackupFile, recordType); persist.recover(); + /* }else{ logger.info("Cannot delete/insert document before {} and after {}.", AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistStartTime), AccountingAggregatorPlugin.LOCAL_TIME_DATE_FORMAT.format(persistEndTime)); } + */ } protected File getOriginalRecordsBackupFile(File elaborationDirectory, String name) throws Exception { diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java index 99e281f..cb12962 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java @@ -10,6 +10,7 @@ import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin.Elabora import org.gcube.accounting.aggregator.utility.Utility; import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.StorageStatusRecord; import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord; import org.junit.Test; import org.slf4j.Logger; @@ -77,9 +78,64 @@ public class AccountingAggregatorPluginTest extends ContextTest { } + @Test + public void aggregateStorageStatus() throws Exception { + //ContextTest.setContextByName(ROOT_DEV_SCOPE); + ContextTest.setContextByName(ROOT_PROD); + + Map inputs = new HashMap(); + + + AggregationType aggregationType = AggregationType.MONTHLY; + //type aggregation + inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name()); + + inputs.put(AccountingAggregatorPlugin.ELABORATION_TYPE_INPUT_PARAMETER, ElaborationType.AGGREGATE.name()); + + + inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(0, 10)); + inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(23, 59)); + + + inputs.put(AccountingAggregatorPlugin.RECORD_TYPE_INPUT_PARAMETER, StorageStatusRecord.class.newInstance().getRecordType()); + + inputs.put(AccountingAggregatorPlugin.RESTART_FROM_LAST_AGGREGATION_DATE_INPUT_PARAMETER, false); + + inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true); + inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); + inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); + + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.MARCH, 1); + String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); + logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + + // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2018, Calendar.JANUARY, 1); + /* + String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); + logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate); + */ + + AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin(); + logger.debug("Going to launch {} with inputs {}", plugin.getName(), inputs); + + while(aggregationStartCalendar.before(aggregationEndCalendar)) { + + plugin.launch(inputs); + Thread.sleep(TimeUnit.SECONDS.toMillis(20)); + plugin = new AccountingAggregatorPlugin(); + aggregationStartCalendar.add(aggregationType.getCalendarField(), 1); + aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); + inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); + } + + } + @Test - public void aggregateStorage() throws Exception { + public void aggregateStorageUsage() throws Exception { //ContextTest.setContextByName(ROOT_DEV_SCOPE); ContextTest.setContextByName(ROOT_PROD); @@ -105,13 +161,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.AUGUST, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2016, Calendar.SEPTEMBER, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2016, Calendar.OCTOBER, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); @@ -161,13 +217,13 @@ public class AccountingAggregatorPluginTest extends ContextTest { inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true); inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true); - Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2019, Calendar.DECEMBER, 1); + Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1); String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate); // Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1); - Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1); + Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.FEBRUARY, 1); /* String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime()); logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate); diff --git a/src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java b/src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java index 637e30f..6f04e71 100644 --- a/src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java +++ b/src/test/java/org/gcube/accounting/aggregator/plugin/MyTest.java @@ -3,6 +3,7 @@ package org.gcube.accounting.aggregator.plugin; import java.text.ParseException; import java.util.Calendar; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -18,9 +19,16 @@ import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedStorageUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord; +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.client.dsl.ListResolver; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.common.storagehub.model.items.Item; import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.documentstore.records.Record; import org.gcube.documentstore.records.RecordUtility; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -278,6 +286,31 @@ public class MyTest { logger.info("{}", now.getTime().after(startTime)); logger.info("{}", now.getTime().before(endTime)); + } + + public static final String ROOT_PROD = "/d4science.research-infrastructures.eu"; + + @Ignore + @Test + public void testStorageHub() throws Exception { + ContextTest.setContextByName(ROOT_PROD); + FolderContainer destinationFolder = null; + StorageHubClient storageHubClient = new StorageHubClient(); + FolderContainer parent = storageHubClient.getWSRoot(); + ListResolver listResolver = parent.findByName("Accounting-Aggregator-Plugin"); + List> itemContainers = listResolver.getContainers(); + if(itemContainers.size()>=1){ + ItemContainer itemContainer = itemContainers.get(0); + if(itemContainer.getType().compareTo(ContainerType.FOLDER)==0) { + destinationFolder = (FolderContainer) itemContainer; + } + } + + logger.debug("{}", destinationFolder); + } + + + } diff --git a/src/test/java/org/gcube/accounting/aggregator/recover/LaunchGenerate.java b/src/test/java/org/gcube/accounting/aggregator/recover/LaunchGenerate.java index df2c521..1ce7ce9 100644 --- a/src/test/java/org/gcube/accounting/aggregator/recover/LaunchGenerate.java +++ b/src/test/java/org/gcube/accounting/aggregator/recover/LaunchGenerate.java @@ -5,6 +5,8 @@ import java.util.ArrayList; import java.util.List; import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,7 +21,8 @@ public class LaunchGenerate { @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(LaunchGenerate.class); - //@Test + @Ignore + @Test public void run() throws Exception { File src = new File("src"); File test = new File(src, "test"); @@ -40,4 +43,16 @@ public class LaunchGenerate { } + @Ignore + @Test + public void testAggregationFromFile() throws Exception { + File src = new File("src"); + File test = new File(src, "test"); + File resources = new File(test, "resources"); + File srcFile = new File(resources, "aux.json"); + + TestAggregationFromFile testAggregationFromFile = new TestAggregationFromFile(srcFile); + testAggregationFromFile.elaborate(); + + } } diff --git a/src/test/java/org/gcube/accounting/aggregator/recover/TestAggregationFromFile.java b/src/test/java/org/gcube/accounting/aggregator/recover/TestAggregationFromFile.java new file mode 100644 index 0000000..9de0bdb --- /dev/null +++ b/src/test/java/org/gcube/accounting/aggregator/recover/TestAggregationFromFile.java @@ -0,0 +1,95 @@ +package org.gcube.accounting.aggregator.recover; + +import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.util.UUID; + +import org.gcube.accounting.aggregator.aggregation.AggregatorBuffer; +import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; +import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; +import org.gcube.documentstore.exception.InvalidValueException; +import org.gcube.documentstore.records.AggregatedRecord; +import org.gcube.documentstore.records.Record; +import org.gcube.documentstore.records.RecordUtility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) + */ +public class TestAggregationFromFile { + + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected static final String ID = Record.ID; + + static { + /// One Record per package is enough + RecordUtility.addRecordPackage(ServiceUsageRecord.class.getPackage()); + RecordUtility.addRecordPackage(AggregatedServiceUsageRecord.class.getPackage()); + } + + protected final File srcFile; + + protected final AggregatorBuffer aggregatorBuffer; + + protected int elaborated; + protected int aggregated; + + public TestAggregationFromFile(File srcFile){ + this.srcFile = srcFile; + this.elaborated = 0; + this.aggregated = 0; + this.aggregatorBuffer = new AggregatorBuffer(); + } + + protected void readFile() throws Exception { + try { + // Open the file that is the first // command line parameter + FileInputStream fstream = new FileInputStream(srcFile); + // Get the object of DataInputStream + DataInputStream in = new DataInputStream(fstream); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + + String line; + // Read File Line By Line + while ((line = br.readLine()) != null) { + elaborateLine(line); + ++elaborated; + } + + br.close(); + in.close(); + fstream.close(); + + } catch (Exception e) { + logger.error("Error while elaborating file {}", srcFile.getAbsolutePath(), e); + throw e; + } + + } + + public void elaborate() throws Exception{ + logger.info("Going to elaborate {}", srcFile.getAbsolutePath()); + readFile(); + } + + protected void elaborateLine(String json) throws Exception { + + Record record = RecordUtility.getRecord(json); + try { + record.validate(); + }catch (InvalidValueException e) { + return; + } + record.setId(UUID.randomUUID().toString()); + + @SuppressWarnings("rawtypes") + AggregatedRecord aggregatedRecord = AggregatorBuffer.getAggregatedRecord(record); + aggregatorBuffer.aggregate(aggregatedRecord); + } + +} diff --git a/src/test/resources/aux.json b/src/test/resources/aux.json new file mode 100644 index 0000000..e38f0b7 --- /dev/null +++ b/src/test/resources/aux.json @@ -0,0 +1,5 @@ +{"operationCount":1,"creationTime":1489770707728,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489770707728,"endTime":1489770707728,"id":"e9b6b523-c89e-4c91-a03f-4796d74ab5b1","operationResult":"SUCCESS"} +{"operationCount":1,"creationTime":1489857068501,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489857068501,"endTime":1489857068501,"id":"28c5315f-9f70-4125-91a4-154d770f733a","operationResult":"SUCCESS"} +{"operationCount":1,"creationTime":1489943468541,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1489943468541,"endTime":1489943468541,"id":"2c8289d5-d016-4684-8045-afa2d1ca8507","operationResult":"SUCCESS"} +{"operationCount":1,"creationTime":1490116270895,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1490116270895,"endTime":1490116270895,"id":"961a213b-8266-4d03-9dc5-8b816c0238c6","operationResult":"SUCCESS"} +{"operationCount":1,"creationTime":1490202675117,"consumerId":"maurizio.tesconi","recordType":"StorageStatusRecord","dataType":"STORAGE","aggregated":true,"dataCount":0,"dataServiceId":"identifier","dataServiceName":"storage-manager","dataVolume":0,"providerId":"MongoDb","scope":"/d4science.research-infrastructures.eu","dataServiceClass":"content-management","startTime":1490202675117,"endTime":1490202675117,"id":"0b906d52-6891-4904-98eb-7b6a4f06d9f0","operationResult":"SUCCESS"} \ No newline at end of file