gCubeSystem
/
accounting-lib
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed Aggregations and added/fixed tests 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-lib@122218 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2016-01-13 15:18:59 +00:00
parent f0de804244
commit 1a48e45faa
9 changed files with 268 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/main/java/org/gcube/accounting/datamodel/aggregation/AggregatedJobUsageRecord.java
View File

@ -6,12 +6,14 @@ package org.gcube.accounting.datamodel.aggregation;
import java.io.Serializable;
import java.util.Calendar;
import java.util.Map;
import java.util.Set;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.accounting.datamodel.basetypes.AbstractJobUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
import org.gcube.documentstore.exception.InvalidValueException;
import org.gcube.documentstore.exception.NotAggregatableRecordsExceptions;
import org.gcube.documentstore.records.aggregation.AggregationUtility;
/**
* This Class is for library internal use only
@ -37,6 +39,11 @@ public class AggregatedJobUsageRecord extends AbstractJobUsageRecord implements
throw new UnsupportedOperationException();
}
@Override
public Set<String> getAggregatedFields() {
return aggregatedFields;
}
@Override
public int getOperationCount() {
return super.getOperationCount();
@ -85,7 +92,17 @@ public class AggregatedJobUsageRecord extends AbstractJobUsageRecord implements
@Override
public AggregatedJobUsageRecord aggregate(AggregatedJobUsageRecord record)
throws NotAggregatableRecordsExceptions {
// TODO Auto-generated method stub
try {
/*
AggregationUtility<AggregatedJobUsageRecord> aggregationUtility = new AggregationUtility<AggregatedJobUsageRecord>(this);
aggregationUtility.aggregate(record);
*/
// TODO
}catch(Exception e){
throw new UnsupportedOperationException(e);
}
//return this;
throw new UnsupportedOperationException();
}
@ -95,10 +112,35 @@ public class AggregatedJobUsageRecord extends AbstractJobUsageRecord implements
@Override
public AggregatedJobUsageRecord aggregate(JobUsageRecord record)
throws NotAggregatableRecordsExceptions {
// TODO Auto-generated method stub
throw new UnsupportedOperationException();
try {
return aggregate(new AggregatedJobUsageRecord(record));
} catch (InvalidValueException e) {
throw new NotAggregatableRecordsExceptions(e.getCause());
}
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(AggregatedJobUsageRecord record) throws NotAggregatableRecordsExceptions {
AggregationUtility<AggregatedJobUsageRecord> aggregationUtility = new AggregationUtility<AggregatedJobUsageRecord>(this);
return aggregationUtility.isAggregable(record);
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(JobUsageRecord record) throws NotAggregatableRecordsExceptions {
try {
return isAggregable(new AggregatedJobUsageRecord(record));
} catch (InvalidValueException e) {
throw new NotAggregatableRecordsExceptions(e.getCause());
}
}
/**
* {@inheritDoc}
*/

48
src/main/java/org/gcube/accounting/datamodel/aggregation/AggregatedPortletUsageRecord.java
View File

@ -6,12 +6,14 @@ package org.gcube.accounting.datamodel.aggregation;
import java.io.Serializable;
import java.util.Calendar;
import java.util.Map;
import java.util.Set;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.accounting.datamodel.basetypes.AbstractPortletUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.PortletUsageRecord;
import org.gcube.documentstore.exception.InvalidValueException;
import org.gcube.documentstore.exception.NotAggregatableRecordsExceptions;
import org.gcube.documentstore.records.aggregation.AggregationUtility;
/**
* This Class is for library internal use only
@ -38,6 +40,11 @@ public class AggregatedPortletUsageRecord extends AbstractPortletUsageRecord imp
throw new UnsupportedOperationException();
}
@Override
public Set<String> getAggregatedFields() {
return aggregatedFields;
}
@Override
public int getOperationCount() {
return super.getOperationCount();
@ -87,7 +94,17 @@ public class AggregatedPortletUsageRecord extends AbstractPortletUsageRecord imp
public AggregatedPortletUsageRecord aggregate(
AggregatedPortletUsageRecord record)
throws NotAggregatableRecordsExceptions {
// TODO Auto-generated method stub
try {
/*
AggregationUtility<AggregatedPortletUsageRecord> aggregationUtility = new AggregationUtility<AggregatedPortletUsageRecord>(this);
aggregationUtility.aggregate(record);
*/
// TODO
}catch(Exception e){
throw new UnsupportedOperationException(e);
}
//return this;
throw new UnsupportedOperationException();
}
@ -97,10 +114,35 @@ public class AggregatedPortletUsageRecord extends AbstractPortletUsageRecord imp
@Override
public AggregatedPortletUsageRecord aggregate(PortletUsageRecord record)
throws NotAggregatableRecordsExceptions {
// TODO Auto-generated method stub
throw new UnsupportedOperationException();
try {
return aggregate(new AggregatedPortletUsageRecord(record));
} catch (InvalidValueException e) {
throw new NotAggregatableRecordsExceptions(e.getCause());
}
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(AggregatedPortletUsageRecord record) throws NotAggregatableRecordsExceptions {
AggregationUtility<AggregatedPortletUsageRecord> aggregationUtility = new AggregationUtility<AggregatedPortletUsageRecord>(this);
return aggregationUtility.isAggregable(record);
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(PortletUsageRecord record) throws NotAggregatableRecordsExceptions {
try {
return isAggregable(new AggregatedPortletUsageRecord(record));
} catch (InvalidValueException e) {
throw new NotAggregatableRecordsExceptions(e.getCause());
}
}
/**
* {@inheritDoc}
*/

41
src/main/java/org/gcube/accounting/datamodel/aggregation/AggregatedServiceUsageRecord.java
View File

@ -6,12 +6,14 @@ package org.gcube.accounting.datamodel.aggregation;
import java.io.Serializable;
import java.util.Calendar;
import java.util.Map;
import java.util.Set;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.accounting.datamodel.basetypes.AbstractServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.documentstore.exception.InvalidValueException;
import org.gcube.documentstore.exception.NotAggregatableRecordsExceptions;
import org.gcube.documentstore.records.aggregation.AggregationUtility;
import org.gcube.documentstore.records.implementation.AggregatedField;
import org.gcube.documentstore.records.implementation.RequiredField;
import org.gcube.documentstore.records.implementation.validations.annotations.ValidLong;
@ -56,6 +58,11 @@ public class AggregatedServiceUsageRecord extends AbstractServiceUsageRecord imp
this.setEndTime(creationTime);
}
@Override
public Set<String> getAggregatedFields() {
return aggregatedFields;
}
@Override
public int getOperationCount() {
return super.getOperationCount();
@ -128,8 +135,10 @@ public class AggregatedServiceUsageRecord extends AbstractServiceUsageRecord imp
public AggregatedServiceUsageRecord aggregate(AggregatedServiceUsageRecord record)
throws NotAggregatableRecordsExceptions {
try {
AggregationUtility<AggregatedServiceUsageRecord> aggregationUtility = new AggregationUtility<AggregatedServiceUsageRecord>(this);
aggregationUtility.aggregate(record);
setDuration(durationWeightedAverage(record));
setOperationCount(this.getOperationCount() + record.getOperationCount());
long max = record.getMaxInvocationTime();
if(max > this.getMaxInvocationTime()){
@ -140,9 +149,10 @@ public class AggregatedServiceUsageRecord extends AbstractServiceUsageRecord imp
if(min < this.getMinInvocationTime()){
this.setMinInvocationTime(min);
}
}catch(Exception e){
throw new UnsupportedOperationException(e);
} catch(NotAggregatableRecordsExceptions e){
throw e;
} catch(Exception ex){
throw new NotAggregatableRecordsExceptions(ex);
}
return this;
@ -161,6 +171,29 @@ public class AggregatedServiceUsageRecord extends AbstractServiceUsageRecord imp
}
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(AggregatedServiceUsageRecord record)
throws NotAggregatableRecordsExceptions {
AggregationUtility<AggregatedServiceUsageRecord> aggregationUtility = new AggregationUtility<AggregatedServiceUsageRecord>(this);
return aggregationUtility.isAggregable(record);
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(ServiceUsageRecord record)
throws NotAggregatableRecordsExceptions {
try {
return isAggregable(new AggregatedServiceUsageRecord(record));
} catch (InvalidValueException e) {
throw new NotAggregatableRecordsExceptions(e.getCause());
}
}
/**
* {@inheritDoc}
*/

34
src/main/java/org/gcube/accounting/datamodel/aggregation/AggregatedStorageUsageRecord.java
View File

@ -6,12 +6,14 @@ package org.gcube.accounting.datamodel.aggregation;
import java.io.Serializable;
import java.util.Calendar;
import java.util.Map;
import java.util.Set;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.accounting.datamodel.basetypes.AbstractStorageUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
import org.gcube.documentstore.exception.InvalidValueException;
import org.gcube.documentstore.exception.NotAggregatableRecordsExceptions;
import org.gcube.documentstore.records.aggregation.AggregationUtility;
import org.gcube.documentstore.records.implementation.AggregatedField;
import org.gcube.documentstore.records.implementation.RequiredField;
import org.gcube.documentstore.records.implementation.validations.annotations.ValidLong;
@ -49,6 +51,11 @@ public class AggregatedStorageUsageRecord extends AbstractStorageUsageRecord imp
this.setEndTime(creationTime);
}
@Override
public Set<String> getAggregatedFields() {
return aggregatedFields;
}
@Override
public int getOperationCount() {
return super.getOperationCount();
@ -99,12 +106,12 @@ public class AggregatedStorageUsageRecord extends AbstractStorageUsageRecord imp
AggregatedStorageUsageRecord record)
throws NotAggregatableRecordsExceptions {
try {
this.setOperationCount(this.getOperationCount() + record.getOperationCount());
AggregationUtility<AggregatedStorageUsageRecord> aggregationUtility = new AggregationUtility<AggregatedStorageUsageRecord>(this);
aggregationUtility.aggregate(record);
this.setDataVolume(this.getDataVolume() + record.getDataVolume());
}catch(Exception e){
throw new UnsupportedOperationException(e);
}
return this;
}
@ -121,6 +128,29 @@ public class AggregatedStorageUsageRecord extends AbstractStorageUsageRecord imp
}
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(AggregatedStorageUsageRecord record)
throws NotAggregatableRecordsExceptions {
AggregationUtility<AggregatedStorageUsageRecord> aggregationUtility = new AggregationUtility<AggregatedStorageUsageRecord>(this);
return aggregationUtility.isAggregable(record);
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(StorageUsageRecord record)
throws NotAggregatableRecordsExceptions {
try {
return isAggregable(new AggregatedStorageUsageRecord(record));
} catch (InvalidValueException e) {
throw new NotAggregatableRecordsExceptions(e.getCause());
}
}
/**
* {@inheritDoc}
*/

55
src/main/java/org/gcube/accounting/datamodel/aggregation/AggregatedTaskUsageRecord.java
View File

@ -6,19 +6,20 @@ package org.gcube.accounting.datamodel.aggregation;
import java.io.Serializable;
import java.util.Calendar;
import java.util.Map;
import java.util.Set;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.accounting.datamodel.basetypes.AbstractTaskUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.TaskUsageRecord;
import org.gcube.documentstore.exception.InvalidValueException;
import org.gcube.documentstore.exception.NotAggregatableRecordsExceptions;
import org.gcube.documentstore.records.aggregation.AggregationUtility;
/**
* This Class is for library internal use only
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
*/
public class AggregatedTaskUsageRecord extends AbstractTaskUsageRecord implements AggregatedUsageRecord<AggregatedTaskUsageRecord, TaskUsageRecord> {
/**
* Generated Serial version UID
@ -32,7 +33,18 @@ public class AggregatedTaskUsageRecord extends AbstractTaskUsageRecord implement
public AggregatedTaskUsageRecord(Map<String, Comparable<? extends Serializable>> properties) throws InvalidValueException{
super(properties);
}
// TODO
public AggregatedTaskUsageRecord(TaskUsageRecord taskUsageRecord) throws InvalidValueException{
throw new UnsupportedOperationException();
}
@Override
public Set<String> getAggregatedFields() {
return aggregatedFields;
}
@Override
public int getOperationCount() {
return super.getOperationCount();
@ -81,7 +93,17 @@ public class AggregatedTaskUsageRecord extends AbstractTaskUsageRecord implement
@Override
public AggregatedTaskUsageRecord aggregate(AggregatedTaskUsageRecord record)
throws NotAggregatableRecordsExceptions {
// TODO Auto-generated method stub
try {
/*
AggregationUtility<AggregatedTaskUsageRecord> aggregationUtility = new AggregationUtility<AggregatedTaskUsageRecord>(this);
aggregationUtility.aggregate(record);
*/
// TODO
}catch(Exception e){
throw new UnsupportedOperationException(e);
}
//return this;
throw new UnsupportedOperationException();
}
@ -91,8 +113,32 @@ public class AggregatedTaskUsageRecord extends AbstractTaskUsageRecord implement
@Override
public AggregatedTaskUsageRecord aggregate(TaskUsageRecord record)
throws NotAggregatableRecordsExceptions {
// TODO Auto-generated method stub
throw new UnsupportedOperationException();
try {
return aggregate(new AggregatedTaskUsageRecord(record));
} catch (InvalidValueException e) {
throw new NotAggregatableRecordsExceptions(e.getCause());
}
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(AggregatedTaskUsageRecord record) throws NotAggregatableRecordsExceptions {
AggregationUtility<AggregatedTaskUsageRecord> aggregationUtility = new AggregationUtility<AggregatedTaskUsageRecord>(this);
return aggregationUtility.isAggregable(record);
}
/**
* {@inheritDoc}
*/
@Override
public boolean isAggregable(TaskUsageRecord record) throws NotAggregatableRecordsExceptions {
try {
return isAggregable(new AggregatedTaskUsageRecord(record));
} catch (InvalidValueException e) {
throw new NotAggregatableRecordsExceptions(e.getCause());
}
}
/**
@ -102,4 +148,5 @@ public class AggregatedTaskUsageRecord extends AbstractTaskUsageRecord implement
public Class<TaskUsageRecord> getAggregable() {
return TaskUsageRecord.class;
}
}

4
src/main/java/org/gcube/accounting/datamodel/basetypes/AbstractJobUsageRecord.java
View File

@ -97,7 +97,7 @@ public abstract class AbstractJobUsageRecord extends BasicUsageRecord {
public Calendar getJobStartTime() {
long millis = (Long) this.resourceProperties.get(JOB_START_TIME);
return timestampStringToCalendar(millis);
return timestampToCalendar(millis);
}
public void setJobStartTime(Calendar jobStartTime) throws InvalidValueException {
@ -106,7 +106,7 @@ public abstract class AbstractJobUsageRecord extends BasicUsageRecord {
public Calendar getJobEndTime() {
long millis = (Long) this.resourceProperties.get(JOB_END_TIME);
return timestampStringToCalendar(millis);
return timestampToCalendar(millis);
}
public void setJobEndTime(Calendar jobEndTime) throws InvalidValueException {

8
src/main/java/org/gcube/accounting/datamodel/basetypes/AbstractTaskUsageRecord.java
View File

@ -121,7 +121,7 @@ public abstract class AbstractTaskUsageRecord extends BasicUsageRecord {
@Deprecated
public Calendar getUsageStartTime() {
long millis = (Long) this.resourceProperties.get(USAGE_START_TIME);
return timestampStringToCalendar(millis);
return timestampToCalendar(millis);
}
@Deprecated
public void setUsageStartTime(Calendar usageStartTime) throws InvalidValueException {
@ -131,7 +131,7 @@ public abstract class AbstractTaskUsageRecord extends BasicUsageRecord {
@Deprecated
public Calendar getUsageEndTime() {
long millis = (Long) this.resourceProperties.get(USAGE_END_TIME);
return timestampStringToCalendar(millis);
return timestampToCalendar(millis);
}
@Deprecated
public void setUsageEndTime(Calendar usageEndTime) throws InvalidValueException {
@ -141,7 +141,7 @@ public abstract class AbstractTaskUsageRecord extends BasicUsageRecord {
public Calendar getTaskStartTime() {
long millis = (Long) this.resourceProperties.get(TASK_START_TIME);
return timestampStringToCalendar(millis);
return timestampToCalendar(millis);
}
public void setTaskStartTime(Calendar startTime) throws InvalidValueException {
@ -151,7 +151,7 @@ public abstract class AbstractTaskUsageRecord extends BasicUsageRecord {
public Calendar getTaskEndTime() {
long millis = (Long) this.resourceProperties.get(TASK_END_TIME);
return timestampStringToCalendar(millis);
return timestampToCalendar(millis);
}
public void setTaskEndTime(Calendar endTime) throws InvalidValueException {

2
src/test/java/org/gcube/accounting/testutility/StressTestUtility.java
View File

@ -18,7 +18,7 @@ public class StressTestUtility {
private static final Logger logger = LoggerFactory.getLogger(StressTestUtility.class);
protected final static int DEFAULT_NUMBER_OF_RECORDS = 3000;
public final static int DEFAULT_NUMBER_OF_RECORDS = 3000;
public static void stressTest(TestOperation operation) throws Exception {
stressTest(operation, DEFAULT_NUMBER_OF_RECORDS);

52
src/test/java/org/gcube/documentstore/records/aggregation/AggregationSchedulerTest.java
View File

@ -3,6 +3,10 @@
*/
package org.gcube.documentstore.records.aggregation;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.gcube.accounting.datamodel.UsageRecord;
import org.gcube.accounting.datamodel.basetypes.TestUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
@ -10,8 +14,9 @@ import org.gcube.accounting.testutility.StressTestUtility;
import org.gcube.accounting.testutility.TestOperation;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.documentstore.persistence.PersistenceExecutor;
import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.records.aggregation.AggregationScheduler;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -39,6 +44,22 @@ public class AggregationSchedulerTest {
};
protected void madeAssertion(Map<String, List<Record>> bufferedRecords, List<String> types, int size, int count) {
for(String type : types){
Assert.assertTrue(bufferedRecords.containsKey(type));
List<Record> records = bufferedRecords.get(type);
Assert.assertTrue(records.size()==size);
for(Record record : records){
Assert.assertTrue(record.getRecordType().compareTo(type)==0);
Assert.assertTrue(record instanceof AggregatedRecord);
@SuppressWarnings("rawtypes")
AggregatedRecord aggregatedRecord = (AggregatedRecord) record;
Assert.assertTrue(aggregatedRecord.getOperationCount()==count);
}
}
}
@Test
public void stressTestAggregableURSingleType() throws Exception {
ScopeProvider.instance.set(TestUsageRecord.TEST_SCOPE);
@ -50,6 +71,10 @@ public class AggregationSchedulerTest {
aggregationScheduler.aggregate(usageRecord, persistenceExecutor);
}
});
List<String> types = new ArrayList<String>();
String serviceUsageRecordtype = TestUsageRecord.createTestServiceUsageRecordAutomaticScope().getRecordType();
types.add(serviceUsageRecordtype);
madeAssertion(aggregationScheduler.bufferedRecords, types, 1, StressTestUtility.DEFAULT_NUMBER_OF_RECORDS);
aggregationScheduler.flush(persistenceExecutor);
}
@ -69,6 +94,12 @@ public class AggregationSchedulerTest {
aggregationScheduler.aggregate(usageRecord, persistenceExecutor);
}
});
List<String> types = new ArrayList<String>();
String serviceUsageRecordtype = TestUsageRecord.createTestServiceUsageRecordAutomaticScope().getRecordType();
types.add(serviceUsageRecordtype);
madeAssertion(aggregationScheduler.bufferedRecords, types, 2, StressTestUtility.DEFAULT_NUMBER_OF_RECORDS/2);
aggregationScheduler.flush(persistenceExecutor);
}
@ -88,6 +119,15 @@ public class AggregationSchedulerTest {
aggregationScheduler.aggregate(usageRecord, persistenceExecutor);
}
});
List<String> types = new ArrayList<String>();
String serviceUsageRecordtype = TestUsageRecord.createTestServiceUsageRecordAutomaticScope().getRecordType();
String storageUsageRecordtype = TestUsageRecord.createTestStorageUsageRecordAutomaticScope().getRecordType();
types.add(serviceUsageRecordtype);
types.add(storageUsageRecordtype);
madeAssertion(aggregationScheduler.bufferedRecords, types, 1, StressTestUtility.DEFAULT_NUMBER_OF_RECORDS/2);
aggregationScheduler.flush(persistenceExecutor);
}
@ -116,6 +156,16 @@ public class AggregationSchedulerTest {
aggregationScheduler.aggregate(usageRecord, persistenceExecutor);
}
});
/*
List<String> types = new ArrayList<String>();
String serviceUsageRecordtype = TestUsageRecord.createTestServiceUsageRecordAutomaticScope().getRecordType();
String storageUsageRecordtype = TestUsageRecord.createTestStorageUsageRecordAutomaticScope().getRecordType();
types.add(serviceUsageRecordtype);
types.add(storageUsageRecordtype);
madeAssertion(aggregationScheduler.bufferedRecords, types, 1, StressTestUtility.DEFAULT_NUMBER_OF_RECORDS/4);
*/
aggregationScheduler.flush(persistenceExecutor);
}