gCubeSystem
/
accounting-analytics
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refs #1746: Separate Accounting Model and generalize solution 

https://support.d4science.org/issues/1746

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics@121991 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2015-12-18 16:13:22 +00:00
parent 8ee5fedccd
commit 2f767dccf8
8 changed files with 73 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
pom.xml
View File

@ -9,7 +9,7 @@
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-analytics</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.1.0-SNAPSHOT</version>
<name>accounting-analytics</name>
<scm>
@ -17,7 +17,7 @@
<developerConnection>scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/${project.artifactId}</developerConnection>
<url>https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/${project.artifactId}</url>
</scm>
<dependencyManagement>
<dependencies>
<dependency>
@ -31,16 +31,22 @@
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.data.publishing</groupId>
<artifactId>document-store-lib</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-lib</artifactId>
<scope>provided</scope>
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
<!-- scope>provided</scope -->
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
@ -58,17 +64,12 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
<version>0.9.10</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20090211</version>
</dependency>
<!-- Test Dependency -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

32
src/main/java/org/gcube/accounting/analytics/ResourceRecordQuery.java
View File

@ -5,6 +5,7 @@ package org.gcube.accounting.analytics;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@ -16,13 +17,12 @@ import org.gcube.accounting.analytics.exception.NoAvailableScopeException;
import org.gcube.accounting.analytics.exception.NoUsableAccountingPersistenceQueryFound;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceBackendQueryFactory;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.accounting.datamodel.SingleUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.records.RecordUtility;
import org.json.JSONException;
import org.json.JSONObject;
import org.reflections.Reflections;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -33,23 +33,21 @@ public class ResourceRecordQuery {
private static Logger logger = LoggerFactory.getLogger(ResourceRecordQuery.class);
protected static Map<Class<? extends SingleUsageRecord>, Set<String>> resourceRecords = null;
protected static Map<Class<? extends Record>, Set<String>> resourceRecords = null;
/**
* Return a Map containing a set of required fields for each Resource
* Records Types
* @return the Map
*/
public static synchronized Map<Class<? extends SingleUsageRecord>, Set<String>> getResourceRecordsTypes() {
public static synchronized Map<Class<? extends Record>, Set<String>> getResourceRecordsTypes() {
if(resourceRecords==null){
resourceRecords = new HashMap<Class<? extends SingleUsageRecord>, Set<String>>();
Package usageRecordPackage = ServiceUsageRecord.class.getPackage();
Reflections reflections = new Reflections(usageRecordPackage.getName());
Set<Class<? extends SingleUsageRecord>> resourceRecordsTypes = reflections.getSubTypesOf(SingleUsageRecord.class);
for(Class<? extends SingleUsageRecord> resourceRecordsType : resourceRecordsTypes){
resourceRecords = new HashMap<Class<? extends Record>, Set<String>>();
Collection<Class<? extends Record>> resourceRecordsTypes = RecordUtility.getRecordClassesFound().values();
for(Class<? extends Record> resourceRecordsType : resourceRecordsTypes){
try {
SingleUsageRecord singleUsageRecord = resourceRecordsType.newInstance();
resourceRecords.put(resourceRecordsType, singleUsageRecord.getRequiredFields());
Record record = resourceRecordsType.newInstance();
resourceRecords.put(resourceRecordsType, record.getRequiredFields());
} catch (InstantiationException | IllegalAccessException e) {
logger.error(String.format("Unable to correctly istantiate %s", resourceRecordsType.getSimpleName()), e);
}
@ -138,7 +136,7 @@ public class ResourceRecordQuery {
* @return the requested list of Info
* @throws Exception if fails
*/
public List<Info> getInfo(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType,
public List<Info> getInfo(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters, boolean pad) throws Exception {
Map<Calendar, Info> unpaddedResults = accountingPersistenceQuery.query(usageRecordType, temporalConstraint, filters);
if(!pad){
@ -155,7 +153,7 @@ public class ResourceRecordQuery {
* @return the requested list of Info
* @throws Exception if fails
*/
public List<Info> getInfo(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType,
public List<Info> getInfo(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{
return getInfo(usageRecordType, temporalConstraint, filters, false);
}
@ -166,14 +164,14 @@ public class ResourceRecordQuery {
* @return a set containing the list of key
* @throws Exception if fails
*/
public List<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType) throws Exception{
public List<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType) throws Exception{
Set<String> keys = accountingPersistenceQuery.getKeys(usageRecordType);
List<String> toSort = new ArrayList<String>(keys);
Collections.sort(toSort);
return toSort;
}
public List<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType, String key) throws Exception {
public List<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType, String key) throws Exception {
Set<String> keys = accountingPersistenceQuery.getPossibleValuesForKey(usageRecordType, key);
List<String> toSort = new ArrayList<String>(keys);
Collections.sort(toSort);

20
src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceBackendQuery.java
View File

@ -11,7 +11,7 @@ import java.util.Set;
import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.documentstore.records.AggregatedRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -25,14 +25,14 @@ public abstract class AccountingPersistenceBackendQuery {
protected abstract void prepareConnection(AccountingPersistenceBackendQueryConfiguration configuration) throws Exception;
protected abstract Map<Calendar, Info> reallyQuery(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType,
protected abstract Map<Calendar, Info> reallyQuery(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception;
/**
* Query the persistence obtaining a Map where the date is the key and
* the #Info is the value. The result is relative to an Usage Record Type,
* respect a TemporalConstraint and can be applied one or more filters.
* @param usageRecordType the Usage Record Type of interest
* @param recordClass the Usage Record Type of interest
* @param temporalConstraint the TemporalConstraint (interval and aggregation)
* @param filters the filter for the query. If null or empty string get all
* data. The filters are evaluated in the order the are presented and are
@ -41,31 +41,31 @@ public abstract class AccountingPersistenceBackendQuery {
* requested data
* @throws Exception if fails
*/
public Map<Calendar, Info> query(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType,
public Map<Calendar, Info> query(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{
logger.trace("Request query: UsageRecordType={}, {}={}, {}s={}", usageRecordType.newInstance().getUsageRecordType(),
logger.trace("Request query: RecordClass={}, {}={}, {}s={}", recordClass.newInstance().getRecordType(),
TemporalConstraint.class.getSimpleName(), temporalConstraint.toString(),
Filter.class.getSimpleName(), filters);
return reallyQuery(usageRecordType, temporalConstraint, filters);
return reallyQuery(recordClass, temporalConstraint, filters);
}
/**
* Return the list of key valid for queries a certain usage record type
* @param usageRecordType the usage record type
* @param recordClass the usage record class
* @return a set containing the list of key
* @throws Exception if fails
*/
public abstract Set<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType) throws Exception;
public abstract Set<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass) throws Exception;
/**
* Return the list of possible values for a key for a certain usage record type
* @param usageRecordType the usage record type
* @param recordClass the usage record type
* @param key the key
* @return a set containing the list of possible values
* @throws Exception if fails
*/
public abstract Set<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType, String key) throws Exception;
public abstract Set<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass, String key) throws Exception;
/**
* Close the connection to persistence

23
src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceBackendQueryConfiguration.java
View File

@ -1,13 +1,9 @@
package org.gcube.accounting.analytics.persistence;
import java.net.URI;
import org.gcube.accounting.persistence.AccountingPersistenceConfiguration;
import org.gcube.common.resources.gcore.ServiceEndpoint;
/**
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
*
*/
public class AccountingPersistenceBackendQueryConfiguration extends AccountingPersistenceConfiguration {
@ -18,22 +14,15 @@ public class AccountingPersistenceBackendQueryConfiguration extends AccountingPe
super();
}
/**
* @param uri the URI of the persistence
* @param username the username to connect to persistence
* @param password the password to connect to persistence
*/
public AccountingPersistenceBackendQueryConfiguration(URI uri, String username, String password){
super(uri, username, password);
}
/**
* @param persistenceClassName The classname of the persistence to instantiate
* @throws Exception if fails
*/
public AccountingPersistenceBackendQueryConfiguration(String persistenceClassName) throws Exception{
super.init();
ServiceEndpoint serviceEndpoint = getServiceEndpoint(SERVICE_ENDPOINT_CATEGORY, SERVICE_ENDPOINT_NAME, persistenceClassName);
setValues(serviceEndpoint, persistenceClassName);
@SuppressWarnings({ "unchecked", "rawtypes" })
public AccountingPersistenceBackendQueryConfiguration(Class<? extends AccountingPersistenceBackendQuery> clz) throws Exception{
super((Class) clz);
}
}

8
src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceBackendQueryFactory.java
View File

@ -45,16 +45,16 @@ public abstract class AccountingPersistenceBackendQueryFactory {
try {
ServiceLoader<AccountingPersistenceBackendQuery> serviceLoader = ServiceLoader.load(AccountingPersistenceBackendQuery.class);
for (AccountingPersistenceBackendQuery found : serviceLoader) {
Class<? extends AccountingPersistenceBackendQuery> foundClass = found.getClass();
try {
String foundClassName = found.getClass().getSimpleName();
String foundClassName = foundClass.getSimpleName();
logger.debug("Testing {}", foundClassName);
AccountingPersistenceBackendQueryConfiguration configuration = new AccountingPersistenceBackendQueryConfiguration(foundClassName);
AccountingPersistenceBackendQueryConfiguration configuration = new AccountingPersistenceBackendQueryConfiguration(foundClass);
found.prepareConnection(configuration);
accountingPersistenceQuery = found;
break;
} catch (Exception e) {
logger.debug(String.format("%s not initialized correctly. It will not be used", found.getClass().getSimpleName()));
logger.debug(String.format("%s not initialized correctly. It will not be used", foundClass.getSimpleName()));
}
}
} catch(Exception e){

16
src/main/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceQuery.java
View File

@ -11,7 +11,7 @@ import java.util.Set;
import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.datamodel.AggregatedUsageRecord;
import org.gcube.documentstore.records.AggregatedRecord;
/**
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
@ -36,7 +36,7 @@ public class AccountingPersistenceQuery {
* Query the persistence obtaining a Map where the date is the key and
* the #Info is the value. The result is relative to an Usage Record Type,
* respect a TemporalConstraint and can be applied one or more filters.
* @param usageRecordType the Usage Record Type of interest
* @param recordClass the Usage Record Type of interest
* @param temporalConstraint the TemporalConstraint (interval and aggregation)
* @param filters the filter for the query. If null or empty string get all
* data. The filters are evaluated in the order the are presented and are
@ -45,19 +45,19 @@ public class AccountingPersistenceQuery {
* requested data
* @throws Exception if fails
*/
public Map<Calendar, Info> query(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType,
public Map<Calendar, Info> query(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass,
TemporalConstraint temporalConstraint, List<Filter> filters) throws Exception{
return AccountingPersistenceBackendQueryFactory.getInstance().query(usageRecordType, temporalConstraint, filters);
return AccountingPersistenceBackendQueryFactory.getInstance().query(recordClass, temporalConstraint, filters);
}
/**
* Return the list of key valid for queries a certain usage record type
* @param usageRecordType the usage record type
* @param recordClass the usage record type
* @return a set containing the list of key
* @throws Exception if fails
*/
public Set<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType) throws Exception {
return AccountingPersistenceBackendQueryFactory.getInstance().getKeys(usageRecordType);
public Set<String> getKeys(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> recordClass) throws Exception {
return AccountingPersistenceBackendQueryFactory.getInstance().getKeys(recordClass);
}
@ -68,7 +68,7 @@ public class AccountingPersistenceQuery {
* @return a set containing the list of possible values
* @throws Exception if fails
*/
public Set<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedUsageRecord> usageRecordType, String key) throws Exception {
public Set<String> getPossibleValuesForKey(@SuppressWarnings("rawtypes") Class<? extends AggregatedRecord> usageRecordType, String key) throws Exception {
return AccountingPersistenceBackendQueryFactory.getInstance().getPossibleValuesForKey(usageRecordType, key);
}

24
src/test/java/org/gcube/accounting/analytics/ResourceRecordQueryTest.java
View File

@ -7,12 +7,13 @@ import java.util.HashSet;
import java.util.Set;
import org.gcube.accounting.datamodel.BasicUsageRecord;
import org.gcube.accounting.datamodel.SingleUsageRecord;
import org.gcube.accounting.datamodel.RawUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.JobUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.PortletUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.StorageUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.TaskUsageRecord;
import org.gcube.documentstore.records.Record;
import org.junit.Assert;
import org.junit.Test;
@ -21,7 +22,7 @@ import org.junit.Test;
*/
public class ResourceRecordQueryTest {
public class TestUsageRecord extends BasicUsageRecord implements SingleUsageRecord {
public class TestUsageRecord extends BasicUsageRecord {
/**
* Generated Serial Version UID
@ -35,35 +36,32 @@ public class ResourceRecordQueryTest {
}
public static Set<Class<? extends SingleUsageRecord>> getExpectedResourceRecordsTypes(){
Set<Class<? extends SingleUsageRecord>> expected = new HashSet<Class<? extends SingleUsageRecord>>();
public static Set<Class<? extends Record>> getExpectedResourceRecordsTypes(){
Set<Class<? extends Record>> expected = new HashSet<Class<? extends Record>>();
expected.add(ServiceUsageRecord.class);
expected.add(StorageUsageRecord.class);
expected.add(JobUsageRecord.class);
expected.add(TaskUsageRecord.class);
expected.add(PortletUsageRecord.class);
expected.add(RawUsageRecord.class);
return expected;
}
@Test
public void testGetResourceRecordsTypes(){
Set<Class<? extends SingleUsageRecord>> expected = getExpectedResourceRecordsTypes();
Set<Class<? extends SingleUsageRecord>> found = ResourceRecordQuery.getResourceRecordsTypes().keySet();
Set<Class<? extends Record>> expected = getExpectedResourceRecordsTypes();
Set<Class<? extends Record>> found = ResourceRecordQuery.getResourceRecordsTypes().keySet();
Assert.assertTrue(expected.containsAll(found));
Assert.assertTrue(found.containsAll(expected));
}
@Test
public void testGetResourceRecordsTypesWithFakeClass(){
Set<Class<? extends SingleUsageRecord>> expected = getExpectedResourceRecordsTypes();
Set<Class<? extends Record>> expected = getExpectedResourceRecordsTypes();
expected.add(TestUsageRecord.class);
Set<Class<? extends SingleUsageRecord>> found = ResourceRecordQuery.getResourceRecordsTypes().keySet();
Set<Class<? extends Record>> found = ResourceRecordQuery.getResourceRecordsTypes().keySet();
Assert.assertTrue(expected.containsAll(found));
Assert.assertFalse(found.containsAll(expected));
}
}

12
src/test/java/org/gcube/accounting/analytics/persistence/AccountingPersistenceConfigurationTest.java
View File

@ -21,16 +21,18 @@ public class AccountingPersistenceConfigurationTest {
public static final String[] SCOPES = new String[]{"/gcube", "/gcube/devNext", "/gcube/devsec"};
public abstract class AccountingPersistenceQueryCouchDB extends AccountingPersistenceBackendQuery {}
@Test
public void getUsernamePasswordForScopes() throws Exception{
for(String scope : SCOPES){
ScopeProvider.instance.set(scope);
try {
AccountingPersistenceConfiguration persitenceConfiguration = new AccountingPersistenceConfiguration(COUCHDB_CLASS_NAME);
logger.debug("{} {} - {} : {}", scope,
persitenceConfiguration.getUri(),
persitenceConfiguration.getUsername(),
persitenceConfiguration.getPassword());
AccountingPersistenceBackendQueryConfiguration persitenceConfiguration = new AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchDB.class);
String uri = persitenceConfiguration.getProperty(AccountingPersistenceConfiguration.URL_PROPERTY_KEY);
String username = persitenceConfiguration.getProperty(AccountingPersistenceConfiguration.USERNAME_PROPERTY_KEY);
String password = persitenceConfiguration.getProperty(AccountingPersistenceConfiguration.PASSWORD_PROPERTY_KEY);
logger.debug("{} {} - {} : {}", scope, uri, username, password);
}catch(IndexOutOfBoundsException e){
logger.debug("No AccountingPersistenceConfiguration : \n {} {} \n\n", e.getClass().getName(), e.getMessage());
} catch(Exception e){