diff --git a/src/main/java/org/gcube/accounting/aggregation/ServiceUsageRecord.java b/src/main/java/org/gcube/accounting/aggregation/ServiceUsageRecord.java index 6eafcad..b317452 100644 --- a/src/main/java/org/gcube/accounting/aggregation/ServiceUsageRecord.java +++ b/src/main/java/org/gcube/accounting/aggregation/ServiceUsageRecord.java @@ -47,7 +47,7 @@ public class ServiceUsageRecord extends org.gcube.accounting.datamodel.basetypes init(); } - public ServiceUsageRecord(org.gcube.accounting.datamodel.basetypes.ServiceUsageRecord record) throws InvalidValueException{ + public ServiceUsageRecord(org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord record) throws InvalidValueException{ super(record.getResourceProperties()); this.setOperationCount(1); long duration = record.getDuration(); diff --git a/src/main/java/org/gcube/accounting/aggregation/StorageUsageRecord.java b/src/main/java/org/gcube/accounting/aggregation/StorageUsageRecord.java index a79fa38..607b4c0 100644 --- a/src/main/java/org/gcube/accounting/aggregation/StorageUsageRecord.java +++ b/src/main/java/org/gcube/accounting/aggregation/StorageUsageRecord.java @@ -42,7 +42,7 @@ public class StorageUsageRecord extends org.gcube.accounting.datamodel.basetypes init(); } - public StorageUsageRecord(org.gcube.accounting.datamodel.basetypes.StorageUsageRecord record) throws InvalidValueException{ + public StorageUsageRecord(org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord record) throws InvalidValueException{ super(record.getResourceProperties()); this.setOperationCount(1); Calendar creationTime = record.getCreationTime(); diff --git a/src/main/java/org/gcube/accounting/datamodel/AggregationStrategy.java b/src/main/java/org/gcube/accounting/datamodel/AggregationStrategy.java index 4a9f74e..d03d633 100644 --- a/src/main/java/org/gcube/accounting/datamodel/AggregationStrategy.java +++ b/src/main/java/org/gcube/accounting/datamodel/AggregationStrategy.java @@ -21,13 +21,27 @@ public abstract class AggregationStrategy, protected T t; protected Set aggregationField; + protected void cleanExtraFields(){ + Set requiredFields = ((BasicUsageRecord) t).getRequiredFields(); + Set keysToRemove = new HashSet(); + for(String propertyName : t.getResourceProperties().keySet()){ + if(!requiredFields.contains(propertyName)){ + keysToRemove.add(propertyName); + } + } + for(String keyToRemove : keysToRemove){ + t.getResourceProperties().remove(keyToRemove); + } + } + public AggregationStrategy(T t){ this.t = t; this.aggregationField = new HashSet(); this.aggregationField.add(BasicUsageRecord.CONSUMER_ID); this.aggregationField.add(BasicUsageRecord.USAGE_RECORD_TYPE); this.aggregationField.add(BasicUsageRecord.SCOPE); - this.aggregationField.add(BasicUsageRecord.OPERATION_RESULT); + this.aggregationField.add(BasicUsageRecord.OPERATION_RESULT); + //cleanExtraFields(); } public T getAggregatedUsageRecord(){ diff --git a/src/main/java/org/gcube/accounting/datamodel/BasicUsageRecord.java b/src/main/java/org/gcube/accounting/datamodel/BasicUsageRecord.java index 12bc062..bf788d0 100644 --- a/src/main/java/org/gcube/accounting/datamodel/BasicUsageRecord.java +++ b/src/main/java/org/gcube/accounting/datamodel/BasicUsageRecord.java @@ -92,6 +92,13 @@ public abstract class BasicUsageRecord implements UsageRecord, Serializable { protected Map> validation; protected Set requiredFields; + /** + * @return the requiredFields + */ + public Set getRequiredFields() { + return requiredFields; + } + protected Set computedFields; /** diff --git a/src/main/java/org/gcube/accounting/persistence/Persistence.java b/src/main/java/org/gcube/accounting/persistence/Persistence.java index 7de397a..5d3f993 100644 --- a/src/main/java/org/gcube/accounting/persistence/Persistence.java +++ b/src/main/java/org/gcube/accounting/persistence/Persistence.java @@ -10,7 +10,6 @@ import java.util.concurrent.Executors; import org.gcube.accounting.aggregation.scheduler.AggregationScheduler; import org.gcube.accounting.datamodel.SingleUsageRecord; -import org.gcube.accounting.datamodel.TestUsageRecord; import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.exception.InvalidValueException; import org.slf4j.Logger; @@ -75,7 +74,7 @@ public abstract class Persistence { if(persistence==null){ persistence = fallback; } - persistence.accountWithFallback(TestUsageRecord.createTestServiceUsageRecord()); + //persistence.accountWithFallback(TestUsageRecord.createTestServiceUsageRecord()); } catch(Exception e){ logger.error("Unable to instance a Persistence Implementation. Using fallback as default", e.getCause()); diff --git a/src/test/java/org/gcube/accounting/aggregation/scheduler/AggregationSchedulerTest.java b/src/test/java/org/gcube/accounting/aggregation/scheduler/AggregationSchedulerTest.java index 47afc79..0db0d08 100644 --- a/src/test/java/org/gcube/accounting/aggregation/scheduler/AggregationSchedulerTest.java +++ b/src/test/java/org/gcube/accounting/aggregation/scheduler/AggregationSchedulerTest.java @@ -4,12 +4,12 @@ package org.gcube.accounting.aggregation.scheduler; import org.gcube.accounting.datamodel.SingleUsageRecord; -import org.gcube.accounting.datamodel.TestUsageRecord; import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.persistence.PersistenceExecutor; import org.gcube.accounting.testutility.StressTestUtility; import org.gcube.accounting.testutility.TestOperation; +import org.gcube.accounting.testutility.TestUsageRecord; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +30,9 @@ public class AggregationSchedulerTest { @Override public void persist(UsageRecord... usageRecords) throws Exception { - logger.debug(usageRecords.toString()); + for(UsageRecord usageRecord : usageRecords){ + logger.debug(usageRecord.toString()); + } } }; @@ -49,7 +51,7 @@ public class AggregationSchedulerTest { aggregationScheduler.flush(persistenceExecutor); } - public static final String ALTERNATIVE_SERVICE_CLASS = "ContentManagement"; + public static final String ALTERNATIVE_SERVICE_CLASS = "AlternativeServiceClass"; @Test public void stressTestDifferentAggregableURSingleType() throws Exception { @@ -85,17 +87,34 @@ public class AggregationSchedulerTest { aggregationScheduler.flush(persistenceExecutor); } - /* @Test public void stressTestDifferentAggregableURMultipleType() throws Exception { + final AggregationScheduler aggregationScheduler = getAggregationScheduler(); StressTestUtility.stressTest(new TestOperation() { @Override public void operate(int i) throws Exception { - + SingleUsageRecord usageRecord; + switch (i%3) { + case 0: + usageRecord = TestUsageRecord.createTestServiceUsageRecord(); + break; + case 1: + usageRecord = TestUsageRecord.createTestStorageUsageRecord(); + break; + case 2: + usageRecord = TestUsageRecord.createTestJobUsageRecord(); + break; + default: + usageRecord = TestUsageRecord.createTestJobUsageRecord(); + } + + aggregationScheduler.aggregate(usageRecord, persistenceExecutor); } }); + aggregationScheduler.flush(persistenceExecutor); } + /* @Test public void stressTestDifferentAggregableAndNotAggregable() throws Exception { StressTestUtility.stressTest(new TestOperation() { diff --git a/src/test/java/org/gcube/accounting/aggregation/strategy/ServiceUsageRecordAggregationStrategyTest.java b/src/test/java/org/gcube/accounting/aggregation/strategy/ServiceUsageRecordAggregationStrategyTest.java index 7eef151..c36c853 100644 --- a/src/test/java/org/gcube/accounting/aggregation/strategy/ServiceUsageRecordAggregationStrategyTest.java +++ b/src/test/java/org/gcube/accounting/aggregation/strategy/ServiceUsageRecordAggregationStrategyTest.java @@ -4,10 +4,10 @@ package org.gcube.accounting.aggregation.strategy; import org.gcube.accounting.aggregation.strategy.ServiceUsageRecordAggregationStrategy; -import org.gcube.accounting.datamodel.TestUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.exception.InvalidValueException; import org.gcube.accounting.exception.NotAggregatableRecordsExceptions; +import org.gcube.accounting.testutility.TestUsageRecord; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -26,6 +26,7 @@ public class ServiceUsageRecordAggregationStrategyTest { public void secondAsNotAggregated() throws InvalidValueException, NotAggregatableRecordsExceptions { ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord(); + serviceUsageRecord.setResourceProperty(TestUsageRecord.TEST_PROPERTY_NAME, TestUsageRecord.TEST_PROPERTY_VALUE); serviceUsageRecord.validate(); logger.debug("ServiceUsageRecord : {}", serviceUsageRecord); @@ -57,6 +58,8 @@ public class ServiceUsageRecordAggregationStrategyTest { Assert.assertTrue(aggregated.getMinInvocationTime() == firstDuration); } + Assert.assertFalse(aggregated.getResourceProperties().containsKey(TestUsageRecord.TEST_PROPERTY_NAME)); + } @@ -73,6 +76,7 @@ public class ServiceUsageRecordAggregationStrategyTest { aggregated.validate(); ServiceUsageRecord serviceUsageRecord2 = TestUsageRecord.createTestServiceUsageRecord(); + serviceUsageRecord2.setResourceProperty(TestUsageRecord.TEST_PROPERTY_NAME, TestUsageRecord.TEST_PROPERTY_VALUE); serviceUsageRecord2.validate(); logger.debug("ServiceUsageRecord 2 : {}", serviceUsageRecord2); org.gcube.accounting.aggregation.ServiceUsageRecord converted = @@ -99,12 +103,14 @@ public class ServiceUsageRecordAggregationStrategyTest { Assert.assertTrue(aggregated.getMaxInvocationTime() == secondDuration); Assert.assertTrue(aggregated.getMinInvocationTime() == firstduration); } + Assert.assertFalse(aggregated.getResourceProperties().containsKey(TestUsageRecord.TEST_PROPERTY_NAME)); } @Test public void aggregationStressTest() throws InvalidValueException, NotAggregatableRecordsExceptions { ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord(); + serviceUsageRecord.setResourceProperty(TestUsageRecord.TEST_PROPERTY_NAME, TestUsageRecord.TEST_PROPERTY_VALUE); serviceUsageRecord.validate(); logger.debug("ServiceUsageRecord : {}", serviceUsageRecord); @@ -118,10 +124,10 @@ public class ServiceUsageRecordAggregationStrategyTest { for(int i=2; i<1002; i++){ ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord(); + sur.setResourceProperty(TestUsageRecord.TEST_PROPERTY_NAME, TestUsageRecord.TEST_PROPERTY_VALUE); sur.validate(); logger.debug("Cycle ServiceUsageRecord {}: {}", i, sur); - long minInvocationTime = aggregated.getMinInvocationTime(); long maxInvocationTime = aggregated.getMaxInvocationTime(); long oldDuration = aggregated.getDuration(); @@ -146,6 +152,7 @@ public class ServiceUsageRecordAggregationStrategyTest { Assert.assertTrue(aggregated.getMaxInvocationTime() == newDuration); } + Assert.assertFalse(aggregated.getResourceProperties().containsKey(TestUsageRecord.TEST_PROPERTY_NAME)); } logger.debug("Resulting Aggregated ServiceUsageRecord: {}", aggregated); diff --git a/src/test/java/org/gcube/accounting/aggregation/strategy/StorageUsageRecordAggregationStrategyTest.java b/src/test/java/org/gcube/accounting/aggregation/strategy/StorageUsageRecordAggregationStrategyTest.java index 6b4ba90..dfff38e 100644 --- a/src/test/java/org/gcube/accounting/aggregation/strategy/StorageUsageRecordAggregationStrategyTest.java +++ b/src/test/java/org/gcube/accounting/aggregation/strategy/StorageUsageRecordAggregationStrategyTest.java @@ -4,10 +4,10 @@ package org.gcube.accounting.aggregation.strategy; import org.gcube.accounting.aggregation.strategy.StorageUsageRecordAggregationStrategy; -import org.gcube.accounting.datamodel.TestUsageRecord; import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord; import org.gcube.accounting.exception.InvalidValueException; import org.gcube.accounting.exception.NotAggregatableRecordsExceptions; +import org.gcube.accounting.testutility.TestUsageRecord; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -26,6 +26,7 @@ public class StorageUsageRecordAggregationStrategyTest { public void secondAsNotAggregated() throws InvalidValueException, NotAggregatableRecordsExceptions { StorageUsageRecord storageUsageRecord = TestUsageRecord.createTestStorageUsageRecord(); + storageUsageRecord.setResourceProperty(TestUsageRecord.TEST_PROPERTY_NAME, TestUsageRecord.TEST_PROPERTY_VALUE); storageUsageRecord.validate(); logger.debug("StorageUsageRecord : {}", storageUsageRecord); @@ -49,6 +50,7 @@ public class StorageUsageRecordAggregationStrategyTest { Assert.assertTrue(aggregated.getDataVolume() == (firstDataVolume + secondDataVolume)); Assert.assertTrue(aggregated.getOperationCount() == 2); + Assert.assertFalse(aggregated.getResourceProperties().containsKey(TestUsageRecord.TEST_PROPERTY_NAME)); } @Test @@ -64,6 +66,7 @@ public class StorageUsageRecordAggregationStrategyTest { aggregated.validate(); StorageUsageRecord storageUsageRecord2 = TestUsageRecord.createTestStorageUsageRecord(); + storageUsageRecord2.setResourceProperty(TestUsageRecord.TEST_PROPERTY_NAME, TestUsageRecord.TEST_PROPERTY_VALUE); storageUsageRecord2.validate(); logger.debug("StorageUsageRecord 2 : {}", storageUsageRecord2); org.gcube.accounting.aggregation.StorageUsageRecord converted = @@ -83,13 +86,14 @@ public class StorageUsageRecordAggregationStrategyTest { Assert.assertTrue(aggregated.getDataVolume() == (firstDataVolume + secondDataVolume)); Assert.assertTrue(aggregated.getOperationCount() == 2); - + Assert.assertFalse(aggregated.getResourceProperties().containsKey(TestUsageRecord.TEST_PROPERTY_NAME)); } @Test public void aggregationStressTest() throws InvalidValueException, NotAggregatableRecordsExceptions { StorageUsageRecord storageUsageRecord = TestUsageRecord.createTestStorageUsageRecord(); + storageUsageRecord.setResourceProperty(TestUsageRecord.TEST_PROPERTY_NAME, TestUsageRecord.TEST_PROPERTY_VALUE); storageUsageRecord.validate(); logger.debug("StorageUsageRecord : {}", storageUsageRecord); @@ -103,6 +107,7 @@ public class StorageUsageRecordAggregationStrategyTest { for(int i=2; i<1002; i++){ StorageUsageRecord sur = TestUsageRecord.createTestStorageUsageRecord(); + sur.setResourceProperty(TestUsageRecord.TEST_PROPERTY_NAME, TestUsageRecord.TEST_PROPERTY_VALUE); sur.validate(); logger.debug("Cycle StorageUsageRecord {}: {}", i, sur); @@ -115,7 +120,7 @@ public class StorageUsageRecordAggregationStrategyTest { Assert.assertTrue(aggregated.getDataVolume() == (oldDataVolume + newDataVolume)); Assert.assertTrue(aggregated.getOperationCount() == i); - + Assert.assertFalse(aggregated.getResourceProperties().containsKey(TestUsageRecord.TEST_PROPERTY_NAME)); } logger.debug("Resulting Aggregated StorageUsageRecord: {}", aggregated); diff --git a/src/test/java/org/gcube/accounting/datamodel/UsageRecordTest.java b/src/test/java/org/gcube/accounting/datamodel/UsageRecordTest.java index 6fcdd0f..a02d213 100644 --- a/src/test/java/org/gcube/accounting/datamodel/UsageRecordTest.java +++ b/src/test/java/org/gcube/accounting/datamodel/UsageRecordTest.java @@ -4,6 +4,7 @@ package org.gcube.accounting.datamodel; import org.gcube.accounting.exception.InvalidValueException; +import org.gcube.accounting.testutility.TestUsageRecord; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/gcube/accounting/persistence/PersistenceTest.java b/src/test/java/org/gcube/accounting/persistence/PersistenceTest.java index 921b4ab..b72c348 100644 --- a/src/test/java/org/gcube/accounting/persistence/PersistenceTest.java +++ b/src/test/java/org/gcube/accounting/persistence/PersistenceTest.java @@ -4,9 +4,9 @@ package org.gcube.accounting.persistence; import org.gcube.accounting.datamodel.SingleUsageRecord; -import org.gcube.accounting.datamodel.TestUsageRecord; import org.gcube.accounting.testutility.StressTestUtility; import org.gcube.accounting.testutility.TestOperation; +import org.gcube.accounting.testutility.TestUsageRecord; import org.junit.Test; /** diff --git a/src/main/java/org/gcube/accounting/datamodel/TestUsageRecord.java b/src/test/java/org/gcube/accounting/testutility/TestUsageRecord.java similarity index 87% rename from src/main/java/org/gcube/accounting/datamodel/TestUsageRecord.java rename to src/test/java/org/gcube/accounting/testutility/TestUsageRecord.java index 7c511ed..b770fa7 100644 --- a/src/main/java/org/gcube/accounting/datamodel/TestUsageRecord.java +++ b/src/test/java/org/gcube/accounting/testutility/TestUsageRecord.java @@ -1,12 +1,13 @@ /** * */ -package org.gcube.accounting.datamodel; +package org.gcube.accounting.testutility; import java.net.URI; import java.net.URISyntaxException; import org.gcube.accounting.datamodel.UsageRecord.OperationResult; +import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord; import org.gcube.accounting.datamodel.basetypes.StorageUsageRecord.DataType; @@ -66,7 +67,7 @@ public class TestUsageRecord { usageRecord.setDuration(generateRandomLong(MIN_DURATION, MAX_DURATION)); - usageRecord.setResourceProperty(TEST_PROPERTY_NAME, TEST_PROPERTY_VALUE); + } catch (InvalidValueException e) { logger.error(" ------ You SHOULD NOT SEE THIS MESSAGE. Error Creating a test Usage Record", e.getCause()); @@ -106,8 +107,7 @@ public class TestUsageRecord { usageRecord.setDataVolume(generateRandomLong(MIN_DATA_VOLUME, MAX_DATA_VOLUME)); usageRecord.setQualifier("image/png"); - - usageRecord.setResourceProperty(TEST_PROPERTY_NAME, TEST_PROPERTY_VALUE); + } catch (InvalidValueException | URISyntaxException e) { logger.error(" ------ You SHOULD NOT SEE THIS MESSAGE. Error Creating a test Usage Record", e.getCause()); @@ -115,4 +115,22 @@ public class TestUsageRecord { return usageRecord; } + + /** + * @return + */ + public static JobUsageRecord createTestJobUsageRecord() { + + JobUsageRecord usageRecord = new JobUsageRecord(); + try { + usageRecord.setConsumerId(TEST_CONSUMER_ID); + usageRecord.setScope(TEST_SCOPE); + usageRecord.setOperationResult(OperationResult.SUCCESS); + + + } catch (InvalidValueException e) { + logger.error(" ------ You SHOULD NOT SEE THIS MESSAGE. Error Creating a test Usage Record", e.getCause()); + } + return usageRecord; + } }