Refs #10188: Add a way to esplicitly set the scope in accounting-analytics

Task-Url: https://support.d4science.org/issues/10188

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-analytics-persistence-couchbase@161933 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-01-02 16:22:13 +00:00
parent 407f5d1d17
commit 2edcb5a076
3 changed files with 666 additions and 780 deletions

View File

@ -1,8 +1,4 @@
package org.gcube.accounting.analytics.persistence.couchbase; package org.gcube.accounting.analytics.persistence.couchbase;
/**
*
*/
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
@ -40,61 +36,56 @@ import org.slf4j.LoggerFactory;
* *
*/ */
public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest { public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest {
private static Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryCouchBaseTest.class); private static Logger logger = LoggerFactory.getLogger(AccountingPersistenceQueryCouchBaseTest.class);
protected AccountingPersistenceQueryCouchBase accountingPersistenceQueryCouchBase; protected AccountingPersistenceQueryCouchBase accountingPersistenceQueryCouchBase;
public class ExtendedInfo extends Info { public class ExtendedInfo extends Info {
protected String key; protected String key;
/** /**
* @return the key * @return the key
*/ */
public String getKey() { public String getKey() {
return key; return key;
} }
/** /**
* @param key the key to set * @param key the key to set
*/ */
public void setKey(String key) { public void setKey(String key) {
this.key = key; this.key = key;
} }
public ExtendedInfo(String key, Calendar calendar, JSONObject value){ public ExtendedInfo(String key, Calendar calendar, JSONObject value) {
super(calendar, value); super(calendar, value);
this.key = key; this.key = key;
} }
public String toString(){ public String toString() {
String info = String .format("Key : %s, %s ", key, super.toString()); String info = String.format("Key : %s, %s ", key, super.toString());
return info; return info;
} }
} }
@Before @Before
public void before() throws Exception{ public void before() throws Exception {
AccountingPersistenceBackendQueryConfiguration configuration = new AccountingPersistenceBackendQueryConfiguration configuration = new AccountingPersistenceBackendQueryConfiguration(
AccountingPersistenceBackendQueryConfiguration(AccountingPersistenceQueryCouchBase.class); AccountingPersistenceQueryCouchBase.class);
accountingPersistenceQueryCouchBase = new AccountingPersistenceQueryCouchBase(); accountingPersistenceQueryCouchBase = new AccountingPersistenceQueryCouchBase();
accountingPersistenceQueryCouchBase.prepareConnection(configuration); accountingPersistenceQueryCouchBase.prepareConnection(configuration);
} }
public void printMap(Map<Calendar,Info> map) {
for(Info info : map.values()) {
public void printMap(Map<Calendar, Info> map){
for(Info info : map.values()){
logger.debug("{}", info); logger.debug("{}", info);
} }
} }
@Test @Test
public void getUsersInVREs() throws Exception { public void getUsersInVREs() throws Exception {
List<String> contexts = new ArrayList<>(); List<String> contexts = new ArrayList<>();
@ -112,25 +103,22 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest {
endTime.set(2017, Calendar.JULY, 15); endTime.set(2017, Calendar.JULY, 15);
logger.debug("EndTime {}", endTime.getTimeInMillis()); logger.debug("EndTime {}", endTime.getTimeInMillis());
TemporalConstraint temporalConstraint =
new TemporalConstraint(startTime.getTimeInMillis(),
endTime.getTimeInMillis(), AggregationMode.MONTHLY);
Class<AggregatedServiceUsageRecord> clz =
AggregatedServiceUsageRecord.class;
TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
endTime.getTimeInMillis(), AggregationMode.MONTHLY);
Class<AggregatedServiceUsageRecord> clz = AggregatedServiceUsageRecord.class;
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
//Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "DataMiner"); //Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "DataMiner");
Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "RConnector"); Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "RConnector");
filters.add(filter); filters.add(filter);
for(String context : contexts) {
for(String context : contexts){
ScopeProvider.instance.set(context); ScopeProvider.instance.set(context);
SortedSet<NumberedFilter> top = accountingPersistenceQueryCouchBase.getNextPossibleValuesWithMap(clz, temporalConstraint, filters, ServiceUsageRecord.CONSUMER_ID, null); SortedSet<NumberedFilter> top = accountingPersistenceQueryCouchBase.getNextPossibleValuesWithMap(clz,
temporalConstraint, filters, ServiceUsageRecord.CONSUMER_ID, null);
logger.info("Context : {} - Users [{}] : {}", context, top.size(), top); logger.info("Context : {} - Users [{}] : {}", context, top.size(), top);
@ -149,102 +137,82 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest {
endTime.set(2017, Calendar.JULY, 15); endTime.set(2017, Calendar.JULY, 15);
logger.debug("EndTime {}", endTime.getTimeInMillis()); logger.debug("EndTime {}", endTime.getTimeInMillis());
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "DataMiner"); Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "DataMiner");
//Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "RConnector"); //Filter filter = new Filter(ServiceUsageRecord.SERVICE_NAME, "RConnector");
filters.add(filter); filters.add(filter);
TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
endTime.getTimeInMillis(), AggregationMode.MONTHLY);
Class<AggregatedServiceUsageRecord> clz = AggregatedServiceUsageRecord.class;
SortedMap<NumberedFilter,SortedMap<Calendar,Info>> set = accountingPersistenceQueryCouchBase.getTopValues(clz,
temporalConstraint, filters, ServiceUsageRecord.CONSUMER_ID, null);
TemporalConstraint temporalConstraint =
new TemporalConstraint(startTime.getTimeInMillis(),
endTime.getTimeInMillis(), AggregationMode.MONTHLY);
Class<AggregatedServiceUsageRecord> clz =
AggregatedServiceUsageRecord.class;
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> set =
accountingPersistenceQueryCouchBase.getTopValues(
clz, temporalConstraint, filters,
ServiceUsageRecord.CONSUMER_ID, null);
logger.debug("Result {}", set); logger.debug("Result {}", set);
} }
@Test @Test
public void testTimeSeries() throws Exception{ public void testTimeSeries() throws Exception {
//AccountingPersistenceQueryFactory.getForcedScope().set("/gcube/devNext/NextNext");
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2015, Calendar.AUGUST, 20); startTime.set(2017, Calendar.NOVEMBER, 29);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
endTime.set(2016, Calendar.SEPTEMBER, 29,23,59); endTime.set(2017, Calendar.DECEMBER, 29);
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
TemporalConstraint temporalConstraint = TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
endTime.getTimeInMillis(), AggregationMode.DAILY); Class<AggregatedJobUsageRecord> clz = AggregatedJobUsageRecord.class;
Class<AggregatedJobUsageRecord> clz = SortedMap<Calendar,Info> set = accountingPersistenceQueryCouchBase.getTimeSeries(clz, temporalConstraint,
AggregatedJobUsageRecord.class; filters);
SortedMap<Calendar, Info> set =
accountingPersistenceQueryCouchBase.getTimeSeries(
clz, temporalConstraint, filters);
logger.debug("Result final{}", set); logger.debug("Result final{}", set);
} }
@Test @Test
public void testTimeSeriesNoContext() throws Exception{ public void testTimeSeriesNoContext() throws Exception {
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2017, Calendar.APRIL, 20); startTime.set(2017, Calendar.APRIL, 20);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
endTime.set(2017, Calendar.APRIL, 29,23,59); endTime.set(2017, Calendar.APRIL, 29, 23, 59);
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
/* /*
Filter filter = Filter filter =
new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService"); new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService");
filters.add(filter); filters.add(filter);
*/ */
TemporalConstraint temporalConstraint = TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
endTime.getTimeInMillis(), AggregationMode.DAILY); Class<AggregatedServiceUsageRecord> clz = AggregatedServiceUsageRecord.class;
Class<AggregatedServiceUsageRecord> clz = SortedMap<Calendar,Info> set = accountingPersistenceQueryCouchBase.getNoContextTimeSeries(clz,
AggregatedServiceUsageRecord.class; temporalConstraint, filters);
SortedMap<Calendar, Info> set =
accountingPersistenceQueryCouchBase.getNoContextTimeSeries(
clz, temporalConstraint, filters);
logger.debug("Result final{}", set); logger.debug("Result final{}", set);
} }
@Test @Test
public void getUsageValue() throws Exception{ public void getUsageValue() throws Exception {
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2015, Calendar.MAY, 1); startTime.set(2015, Calendar.MAY, 1);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
ScopeProvider.instance.set("/gcube"); ScopeProvider.instance.set("/gcube");
Filter filter = Filter filter = new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve");
new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve");
TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
TemporalConstraint temporalConstraint = endTime.getTimeInMillis(), AggregationMode.DAILY);
new TemporalConstraint(startTime.getTimeInMillis(),
endTime.getTimeInMillis(), AggregationMode.DAILY); JSONObject filterValue = accountingPersistenceQueryCouchBase.getUsageValue(AggregatedServiceUsageRecord.class,
temporalConstraint, filter);
JSONObject filterValue =
accountingPersistenceQueryCouchBase.getUsageValue(AggregatedServiceUsageRecord.class, logger.info("result:" + filterValue.toString());
temporalConstraint, filter);
logger.info("result:"+filterValue.toString());
} }
/** /**
* utilizzato per effettuare una singola chiamata ad accounting analytics in modo che ritorni una lista di quote riempite * utilizzato per effettuare una singola chiamata ad accounting analytics in modo che ritorni una lista di quote riempite
* *
@ -278,44 +246,39 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest {
* ] * ]
*/ */
@Test @Test
public void getUsageValueQuotaTotal() throws Exception{ public void getUsageValueQuotaTotal() throws Exception {
String context = ScopedTest.getCurrentScope(DEFAULT_TEST_SCOPE); String context = ScopedTest.getCurrentScope(DEFAULT_TEST_SCOPE);
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2017, Calendar.MAY, 1); startTime.set(2017, Calendar.MAY, 1);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
TemporalConstraint temporalConstraint = TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
endTime.getTimeInMillis(), AggregationMode.DAILY);
/*ask quota for user alessandro pieve /*ask quota for user alessandro pieve
* *
* SERVICE * SERVICE
* */ * */
List<Filter> filters=new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
filters.add(new Filter("serviceClass", "DataAccess")); filters.add(new Filter("serviceClass", "DataAccess"));
filters.add(new Filter("serviceName", "CkanConnector")); filters.add(new Filter("serviceName", "CkanConnector"));
UsageValue totalfilter=new UsageServiceValue(context,"lucio.lelii",AggregatedServiceUsageRecord.class,temporalConstraint,filters); UsageValue totalfilter = new UsageServiceValue(context, "lucio.lelii", AggregatedServiceUsageRecord.class,
temporalConstraint, filters);
/**** /****
*Example call storage status for each consumer id (quota total used ) *Example call storage status for each consumer id (quota total used )
*/ */
//Call quota total for consumerID //Call quota total for consumerID
UsageValue totalfilterStorageStatus=new UsageStorageValue(context,"name.surname",AggregatedStorageStatusRecord.class); UsageValue totalfilterStorageStatus = new UsageStorageValue(context, "name.surname",
UsageValue totalfilterStorageStatus_2=new UsageStorageValue(context,"lucio.lelii",AggregatedStorageStatusRecord.class); AggregatedStorageStatusRecord.class);
UsageValue totalfilterStorageStatus_3=new UsageStorageValue(context,"alessandro.pieve",AggregatedStorageStatusRecord.class); UsageValue totalfilterStorageStatus_2 = new UsageStorageValue(context, "lucio.lelii",
UsageValue totalfilterStorageStatus_4=new UsageStorageValue(context,"giancarlo.panichi",AggregatedStorageStatusRecord.class); AggregatedStorageStatusRecord.class);
UsageValue totalfilterStorageStatus_3 = new UsageStorageValue(context, "alessandro.pieve",
AggregatedStorageStatusRecord.class);
UsageValue totalfilterStorageStatus_4 = new UsageStorageValue(context, "giancarlo.panichi",
AggregatedStorageStatusRecord.class);
/**** /****
*Example call storage status for each consumer id( quota into period) *Example call storage status for each consumer id( quota into period)
@ -327,14 +290,12 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest {
endTimeStorage.set(2017, Calendar.APRIL, 13); endTimeStorage.set(2017, Calendar.APRIL, 13);
/*TemporalConstraint temporalConstraintStorage =new TemporalConstraint(startTimeStorage.getTimeInMillis(), /*TemporalConstraint temporalConstraintStorage =new TemporalConstraint(startTimeStorage.getTimeInMillis(),
endTimeStorage.getTimeInMillis(), AggregationMode.DAILY);*/ endTimeStorage.getTimeInMillis(), AggregationMode.DAILY);*/
TemporalConstraint temporalConstraintStorage =null; TemporalConstraint temporalConstraintStorage = null;
UsageValue totalfilterStorageStatusPeriod=new UsageStorageValue(context,"alessandro.pieve",AggregatedStorageStatusRecord.class,temporalConstraintStorage); UsageValue totalfilterStorageStatusPeriod = new UsageStorageValue(context, "alessandro.pieve",
AggregatedStorageStatusRecord.class, temporalConstraintStorage);
//richiedo la lista di dati usati totali //richiedo la lista di dati usati totali
List<UsageValue> listTotalFilter=new ArrayList<UsageValue>(); List<UsageValue> listTotalFilter = new ArrayList<UsageValue>();
listTotalFilter.add(totalfilter); listTotalFilter.add(totalfilter);
listTotalFilter.add(totalfilterStorageStatus); listTotalFilter.add(totalfilterStorageStatus);
listTotalFilter.add(totalfilterStorageStatus_2); listTotalFilter.add(totalfilterStorageStatus_2);
@ -342,54 +303,44 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest {
listTotalFilter.add(totalfilterStorageStatus_4); listTotalFilter.add(totalfilterStorageStatus_4);
listTotalFilter.add(totalfilterStorageStatusPeriod); listTotalFilter.add(totalfilterStorageStatusPeriod);
logger.info("filterPackageQuota:" + listTotalFilter);
logger.info("filterPackageQuota:"+listTotalFilter); List<UsageValue> filterValue = accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(listTotalFilter);
List<UsageValue> filterValue = logger.info("result:" + filterValue.toString());
accountingPersistenceQueryCouchBase.getUsageValueQuotaTotal(listTotalFilter);
logger.info("result:"+filterValue.toString());
} }
@Test
public void getQuerableKeyJob() throws Exception {
@Test
public void getQuerableKeyJob() throws Exception{
SortedSet<String> keys; SortedSet<String> keys;
keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedJobUsageRecord.class); keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedJobUsageRecord.class);
for (String key : keys) { for(String key : keys) {
if (key != null && !key.isEmpty()) { if(key != null && !key.isEmpty()) {
logger.debug("key:" +key); logger.debug("key:" + key);
} }
} }
logger.debug("List FilterKeys:" + keys.toString()); logger.debug("List FilterKeys:" + keys.toString());
} }
@Test @Test
public void testTopService() throws Exception { public void testTopService() throws Exception {
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2016, Calendar.AUGUST, 27); startTime.set(2016, Calendar.AUGUST, 27);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
endTime.set(2016, Calendar.SEPTEMBER, 28,23,59); endTime.set(2016, Calendar.SEPTEMBER, 28, 23, 59);
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
//filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_NAME, "IS-Registry")); //filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_NAME, "IS-Registry"));
//filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_CLASS, "Common")); //filters.add(new Filter(AggregatedServiceUsageRecord.SERVICE_CLASS, "Common"));
TemporalConstraint temporalConstraint = TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
endTime.getTimeInMillis(), AggregationMode.DAILY);
Class<AggregatedServiceUsageRecord> clz = AggregatedServiceUsageRecord.class;
Class<AggregatedServiceUsageRecord> clz =
AggregatedServiceUsageRecord.class; SortedMap<NumberedFilter,SortedMap<Calendar,Info>> set = accountingPersistenceQueryCouchBase.getTopValues(clz,
temporalConstraint, filters, AggregatedServiceUsageRecord.OPERATION_RESULT, null);
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> set =
accountingPersistenceQueryCouchBase.getTopValues(
clz, temporalConstraint, filters,
AggregatedServiceUsageRecord.OPERATION_RESULT, null);
logger.debug("Result final{}", set); logger.debug("Result final{}", set);
} }
@Test @Test
@ -397,298 +348,250 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest {
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2017, Calendar.FEBRUARY, 1); startTime.set(2017, Calendar.FEBRUARY, 1);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
endTime.set(2017, Calendar.FEBRUARY, 28,23,59); endTime.set(2017, Calendar.FEBRUARY, 28, 23, 59);
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "scarponi")); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "scarponi"));
TemporalConstraint temporalConstraint = TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
endTime.getTimeInMillis(), AggregationMode.DAILY);
Class<AggregatedStorageUsageRecord> clz = AggregatedStorageUsageRecord.class;
Class<AggregatedStorageUsageRecord> clz =
AggregatedStorageUsageRecord.class; SortedMap<NumberedFilter,SortedMap<Calendar,Info>> set = accountingPersistenceQueryCouchBase.getTopValues(clz,
temporalConstraint, filters, AggregatedStorageUsageRecord.OPERATION_TYPE, null);
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> set =
accountingPersistenceQueryCouchBase.getTopValues(
clz, temporalConstraint, filters,
AggregatedStorageUsageRecord.OPERATION_TYPE, null);
logger.debug("Result final{}", set); logger.debug("Result final{}", set);
} }
@Test
public void getQuerableKeyService() throws Exception {
@Test
public void getQuerableKeyService() throws Exception{
SortedSet<String> keys; SortedSet<String> keys;
keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedServiceUsageRecord.class); keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedServiceUsageRecord.class);
for (String key : keys) { for(String key : keys) {
if (key != null && !key.isEmpty()) { if(key != null && !key.isEmpty()) {
logger.debug("key:" +key); logger.debug("key:" + key);
} }
} }
logger.debug("List FilterKeys:" + keys.toString()); logger.debug("List FilterKeys:" + keys.toString());
} }
public static SortedMap<Calendar, Info> padMap( public static SortedMap<Calendar,Info> padMap(SortedMap<Calendar,Info> unpaddedData,
SortedMap<Calendar, Info> unpaddedData,
TemporalConstraint temporalConstraint) throws Exception { TemporalConstraint temporalConstraint) throws Exception {
JSONObject jsonObject = getPaddingJSONObject(unpaddedData); JSONObject jsonObject = getPaddingJSONObject(unpaddedData);
SortedSet<Calendar> sequence = temporalConstraint.getCalendarSequence(); SortedSet<Calendar> sequence = temporalConstraint.getCalendarSequence();
for (Calendar progressTime : sequence) { for(Calendar progressTime : sequence) {
Info info = unpaddedData.get(progressTime); Info info = unpaddedData.get(progressTime);
if (info == null) { if(info == null) {
info = new Info(progressTime, jsonObject); info = new Info(progressTime, jsonObject);
unpaddedData.put(progressTime, info); unpaddedData.put(progressTime, info);
} }
} }
return unpaddedData; return unpaddedData;
} }
public static JSONObject getPaddingJSONObject(
Map<Calendar, Info> unpaddedResults) throws JSONException { public static JSONObject getPaddingJSONObject(Map<Calendar,Info> unpaddedResults) throws JSONException {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
//verify data consistency //verify data consistency
if (unpaddedResults.size()!=0){ if(unpaddedResults.size() != 0) {
Info auxInfo = new ArrayList<Info>(unpaddedResults.values()).get(0); Info auxInfo = new ArrayList<Info>(unpaddedResults.values()).get(0);
JSONObject auxJsonObject = auxInfo.getValue(); JSONObject auxJsonObject = auxInfo.getValue();
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
Iterator<String> keys = auxJsonObject.keys(); Iterator<String> keys = auxJsonObject.keys();
while (keys.hasNext()) { while(keys.hasNext()) {
String key = keys.next(); String key = keys.next();
jsonObject.put(key, 0); jsonObject.put(key, 0);
} }
} }
return jsonObject; return jsonObject;
} }
@Test @Test
public void testFull() throws Exception { public void testFull() throws Exception {
logger.debug("test full "); logger.debug("test full ");
try{ try {
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
//startTime.set(2016, Calendar.AUGUST, 20, 00, 00); //startTime.set(2016, Calendar.AUGUST, 20, 00, 00);
startTime.set(2016, Calendar.AUGUST, 29,00,00); startTime.set(2016, Calendar.AUGUST, 29, 00, 00);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
//endTime.set(2016, Calendar.AUGUST, 29, 23, 59); //endTime.set(2016, Calendar.AUGUST, 29, 23, 59);
endTime.set(2016, Calendar.AUGUST, 31,23,59); endTime.set(2016, Calendar.AUGUST, 31, 23, 59);
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
TemporalConstraint temporalConstraint = TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
endTime.getTimeInMillis(), AggregationMode.DAILY);
Class<AggregatedServiceUsageRecord> clz = AggregatedServiceUsageRecord.class;
Class<AggregatedServiceUsageRecord> clz =
AggregatedServiceUsageRecord.class; SortedMap<NumberedFilter,SortedMap<Calendar,Info>> set = accountingPersistenceQueryCouchBase
.getTopValues(clz, temporalConstraint, filters, AggregatedServiceUsageRecord.CALLED_METHOD, null);
SortedMap<NumberedFilter, SortedMap<Calendar, Info>> set =
accountingPersistenceQueryCouchBase.getTopValues(
clz, temporalConstraint, filters,
AggregatedServiceUsageRecord.CALLED_METHOD, null);
/**pad*/ /**pad*/
int limit=0; int limit = 0;
boolean pad=true; boolean pad = true;
int count = set.size() > limit ? limit : set.size(); int count = set.size() > limit ? limit : set.size();
NumberedFilter firstRemovalKey = null; NumberedFilter firstRemovalKey = null;
logger.debug("set completo"+set.toString()); logger.debug("set completo" + set.toString());
for(NumberedFilter nf : set.keySet()){ for(NumberedFilter nf : set.keySet()) {
if(--count>=0 || limit<=0){ if(--count >= 0 || limit <= 0) {
if(pad){ if(pad) {
padMap(set.get(nf), temporalConstraint); padMap(set.get(nf), temporalConstraint);
} }
}else{ } else {
if(firstRemovalKey==null){ if(firstRemovalKey == null) {
firstRemovalKey = nf; firstRemovalKey = nf;
}else{ } else {
break; break;
} }
} }
} }
if(firstRemovalKey!=null){ if(firstRemovalKey != null) {
logger.debug("First removal key set:"+set.subMap(set.firstKey(), firstRemovalKey)); logger.debug("First removal key set:" + set.subMap(set.firstKey(), firstRemovalKey));
} }
logger.debug("set: " + set);
logger.debug("set: "+set);
} catch(Exception e) {
}catch(Exception e){
e.printStackTrace(); e.printStackTrace();
} }
} }
@Test @Test
public void testContextService() throws Exception { public void testContextService() throws Exception {
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2017, Calendar.APRIL, 20); startTime.set(2017, Calendar.APRIL, 20);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
endTime.set(2017, Calendar.APRIL, 28,23,59); endTime.set(2017, Calendar.APRIL, 28, 23, 59);
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
//filters.add(new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService")); //filters.add(new Filter(AggregatedServiceUsageRecord.CALLED_METHOD, "WebProcessingService"));
//filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); //filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
TemporalConstraint temporalConstraint = TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
endTime.getTimeInMillis(), AggregationMode.DAILY);
Class<AggregatedServiceUsageRecord> clz =
AggregatedServiceUsageRecord.class;
List<String> context=new ArrayList<String>();
Class<AggregatedServiceUsageRecord> clz = AggregatedServiceUsageRecord.class;
List<String> context = new ArrayList<String>();
// , /gcube/devsec/TestCreation12, /gcube/devsec/TestCreation13, /gcube/devsec/TestCreation14, /gcube/devsec/TestCreation15, /gcube/devsec/TestCreation16, /gcube/devsec/TestCreation17, /gcube/devsec/TestCreation7, /gcube/devsec/TestCreation8, /gcube/devsec/TestCreation9, /gcube/devsec/TestLucio2]] // , /gcube/devsec/TestCreation12, /gcube/devsec/TestCreation13, /gcube/devsec/TestCreation14, /gcube/devsec/TestCreation15, /gcube/devsec/TestCreation16, /gcube/devsec/TestCreation17, /gcube/devsec/TestCreation7, /gcube/devsec/TestCreation8, /gcube/devsec/TestCreation9, /gcube/devsec/TestLucio2]]
context.add("/gcube"); context.add("/gcube");
context.add("/gcube/devNext"); context.add("/gcube/devNext");
/* /*
context.add("/gcube/devNext/Luciotest"); context.add("/gcube/devNext/Luciotest");
context.add("/gcube/devNext/NextNext"); context.add("/gcube/devNext/NextNext");
context.add("/gcube/devsec"); context.add("/gcube/devsec");
context.add("/gcube/devsec/SmartCamera"); context.add("/gcube/devsec/SmartCamera");
context.add("/gcube/devsec/statVRE"); context.add("/gcube/devsec/statVRE");
context.add("/gcube/devsec/TestAddLast"); context.add("/gcube/devsec/TestAddLast");
context.add("/gcube/devsec/devVRE"); context.add("/gcube/devsec/devVRE");
context.add("/gcube/devsec/preVRE"); context.add("/gcube/devsec/preVRE");
context.add("/gcube/preprod/preVRE"); context.add("/gcube/preprod/preVRE");
context.add("/gcube/preprod"); context.add("/gcube/preprod");
*/ */
SortedMap<Filter, SortedMap<Calendar, Info>> setContext = SortedMap<Filter,SortedMap<Calendar,Info>> setContext = accountingPersistenceQueryCouchBase
accountingPersistenceQueryCouchBase.getContextTimeSeries( .getContextTimeSeries(clz, temporalConstraint, filters, context);
clz, temporalConstraint, filters,
context);
logger.debug("Result final{}", setContext); logger.debug("Result final{}", setContext);
} }
@Test @Test
public void getListUsage() throws Exception{ public void getListUsage() throws Exception {
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2015, Calendar.SEPTEMBER, 1); startTime.set(2015, Calendar.SEPTEMBER, 1);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
endTime.set(2016, Calendar.OCTOBER, 20,23,59); endTime.set(2016, Calendar.OCTOBER, 20, 23, 59);
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "valentina.marioli")); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "valentina.marioli"));
////TemporalConstraint temporalConstraint = ////TemporalConstraint temporalConstraint =
new TemporalConstraint(startTime.getTimeInMillis(), new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.DAILY);
endTime.getTimeInMillis(), AggregationMode.DAILY);
//String context="/gcube/devNext"; //String context="/gcube/devNext";
List <String>parameters=new ArrayList<String>(); List<String> parameters = new ArrayList<String>();
parameters.add("serviceClass"); parameters.add("serviceClass");
parameters.add("serviceName"); parameters.add("serviceName");
//Class<AggregatedServiceUsageRecord> clz = //Class<AggregatedServiceUsageRecord> clz =
// AggregatedServiceUsageRecord.class; // AggregatedServiceUsageRecord.class;
// SortedMap<String,Integer> result= accountingPersistenceQueryCouchBase.getListUsage(clz,temporalConstraint, filters,context,parameters); // SortedMap<String,Integer> result= accountingPersistenceQueryCouchBase.getListUsage(clz,temporalConstraint, filters,context,parameters);
} }
@Test @Test
public void getRecord() throws Exception{ public void getRecord() throws Exception {
String recordId="91e1c339-d811-45d7-a13f-7385af59e3c8"; String recordId = "91e1c339-d811-45d7-a13f-7385af59e3c8";
String type="service"; String type = "service";
String document = accountingPersistenceQueryCouchBase.getRecord(recordId, type);
logger.debug("document:{}", document);
String document=accountingPersistenceQueryCouchBase.getRecord(recordId, type);
logger.debug("document:{}",document);
} }
@Test
@Test public void getQuerableKeyStorageStatus() throws Exception {
public void getQuerableKeyStorageStatus() throws Exception{
SortedSet<String> keys; SortedSet<String> keys;
keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedStorageStatusRecord.class); keys = AccountingPersistenceQuery.getQuerableKeys(AggregatedStorageStatusRecord.class);
for (String key : keys) { for(String key : keys) {
if (key != null && !key.isEmpty()) { if(key != null && !key.isEmpty()) {
logger.debug("key:" +key); logger.debug("key:" + key);
} }
} }
logger.debug("List FilterKeys:" + keys.toString()); logger.debug("List FilterKeys:" + keys.toString());
} }
@Test @Test
public void testGetSpaceProvidersIds() throws Exception{ public void testGetSpaceProvidersIds() throws Exception {
SortedSet<String> listProvidersId = SortedSet<String> listProvidersId = accountingPersistenceQueryCouchBase.getSpaceProvidersIds();
accountingPersistenceQueryCouchBase.getSpaceProvidersIds();
logger.debug("Result final{}", listProvidersId); logger.debug("Result final{}", listProvidersId);
} }
@Test @Test
public void testGetFilterValue() throws Exception{ public void testGetFilterValue() throws Exception {
//String key="consumerId"; //String key="consumerId";
String key="dataServiceId"; String key = "dataServiceId";
SortedSet<NumberedFilter> listFilterValue = SortedSet<NumberedFilter> listFilterValue = accountingPersistenceQueryCouchBase
accountingPersistenceQueryCouchBase.getFilterValues(AggregatedStorageStatusRecord.class, null, null, key, null); .getFilterValues(AggregatedStorageStatusRecord.class, null, null, key, null);
logger.debug("Result final{}", listFilterValue); logger.debug("Result final{}", listFilterValue);
} }
@Test @Test
public void testUsageStorage() throws Exception { public void testUsageStorage() throws Exception {
Calendar startTime = Calendar.getInstance(); Calendar startTime = Calendar.getInstance();
startTime.set(2017, Calendar.APRIL, 13); startTime.set(2017, Calendar.APRIL, 13);
Calendar endTime = Calendar.getInstance(); Calendar endTime = Calendar.getInstance();
endTime.set(2017, Calendar.APRIL, 26,23,59); endTime.set(2017, Calendar.APRIL, 26, 23, 59);
List<Filter> filters = new ArrayList<Filter>(); List<Filter> filters = new ArrayList<Filter>();
//filters.add(new Filter(AggregatedStorageStatusRecord.DATA_SERVICEID, "identifier")); //filters.add(new Filter(AggregatedStorageStatusRecord.DATA_SERVICEID, "identifier"));
filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve")); filters.add(new Filter(AggregatedServiceUsageRecord.CONSUMER_ID, "alessandro.pieve"));
TemporalConstraint temporalConstraint = TemporalConstraint temporalConstraint = new TemporalConstraint(startTime.getTimeInMillis(),
new TemporalConstraint(startTime.getTimeInMillis(), endTime.getTimeInMillis(), AggregationMode.YEARLY);
endTime.getTimeInMillis(), AggregationMode.YEARLY);
List<String> providerId = new ArrayList<String>();
Class<AggregatedStorageStatusRecord> clz = AggregatedStorageStatusRecord.class;
List<String> providerId=new ArrayList<String>();
Class<AggregatedStorageStatusRecord> clz =
AggregatedStorageStatusRecord.class;
//providerId.add("Rstudio"); //providerId.add("Rstudio");
providerId.add("MongoDb"); providerId.add("MongoDb");
SortedMap<Filter, SortedMap<Calendar, Long>> setContext =
accountingPersistenceQueryCouchBase.getSpaceTimeSeries(
clz,
temporalConstraint, filters,
providerId);
SortedMap<Filter,SortedMap<Calendar,Long>> setContext = accountingPersistenceQueryCouchBase
.getSpaceTimeSeries(clz, temporalConstraint, filters, providerId);
int count = setContext.size(); int count = setContext.size();
Filter firstRemovalKey = null; Filter firstRemovalKey = null;
for(Filter nf : setContext.keySet()){ for(Filter nf : setContext.keySet()) {
if(--count>=0){ if(--count >= 0) {
//if(pad){ //if(pad){
padMapStorage(setContext.get(nf), temporalConstraint); padMapStorage(setContext.get(nf), temporalConstraint);
//} //}
}else{ } else {
if(firstRemovalKey==null){ if(firstRemovalKey == null) {
firstRemovalKey = nf; firstRemovalKey = nf;
}else{ } else {
break; break;
} }
} }
@ -700,32 +603,27 @@ public class AccountingPersistenceQueryCouchBaseTest extends ScopedTest {
} }
*/ */
logger.debug("Result final{}", setContext); logger.debug("Result final{}", setContext);
} }
public SortedMap<Calendar, Long> padMapStorage( public SortedMap<Calendar,Long> padMapStorage(SortedMap<Calendar,Long> unpaddedData,
SortedMap<Calendar, Long> unpaddedData,
TemporalConstraint temporalConstraint) throws Exception { TemporalConstraint temporalConstraint) throws Exception {
//JSONObject jsonObject = getPaddingJSONObject(unpaddedData); //JSONObject jsonObject = getPaddingJSONObject(unpaddedData);
SortedSet<Calendar> sequence = temporalConstraint.getCalendarSequence(); SortedSet<Calendar> sequence = temporalConstraint.getCalendarSequence();
Long longValuePre = null; Long longValuePre = null;
for (Calendar progressTime : sequence) { for(Calendar progressTime : sequence) {
Long longValue = unpaddedData.get(progressTime); Long longValue = unpaddedData.get(progressTime);
if (longValue == null) { if(longValue == null) {
unpaddedData.put(progressTime, longValuePre); unpaddedData.put(progressTime, longValuePre);
}else{ } else {
longValuePre=longValue; longValuePre = longValue;
} }
} }
return unpaddedData; return unpaddedData;
} }
} }

View File

@ -1,6 +1,3 @@
/**
*
*/
package org.gcube.accounting.analytics.persistence.couchbase; package org.gcube.accounting.analytics.persistence.couchbase;
import java.io.IOException; import java.io.IOException;
@ -23,12 +20,9 @@ import org.slf4j.LoggerFactory;
*/ */
public class ScopedTest { public class ScopedTest {
private static final Logger logger = LoggerFactory.getLogger(ScopedTest.class); private static final Logger logger = LoggerFactory.getLogger(ScopedTest.class);
protected static final String PROPERTIES_FILENAME = "token.properties"; protected static final String PROPERTIES_FILENAME = "token.properties";
private static final String GCUBE_DEVNEXT_VARNAME = "GCUBE_DEVNEXT"; private static final String GCUBE_DEVNEXT_VARNAME = "GCUBE_DEVNEXT";
public static final String GCUBE_DEVNEXT; public static final String GCUBE_DEVNEXT;
@ -45,21 +39,20 @@ public class ScopedTest {
public static final String SA_VARNAME = "SA"; public static final String SA_VARNAME = "SA";
public static final String SA; public static final String SA;
public static final String DEFAULT_TEST_SCOPE; public static final String DEFAULT_TEST_SCOPE;
public static final String ALTERNATIVE_TEST_SCOPE; public static final String ALTERNATIVE_TEST_SCOPE;
static { static {
Properties properties = new Properties(); Properties properties = new Properties();
InputStream input = ScopedTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME); InputStream input = ScopedTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME);
try { try {
// load the properties file // load the properties file
properties.load(input); properties.load(input);
} catch (IOException e) { } catch(IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
GCUBE_DEVNEXT = properties.getProperty(GCUBE_DEVNEXT_VARNAME); GCUBE_DEVNEXT = properties.getProperty(GCUBE_DEVNEXT_VARNAME);
GCUBE_DEVNEXT_NEXTNEXT = properties.getProperty(GCUBE_DEVNEXT_NEXTNEXT_VARNAME); GCUBE_DEVNEXT_NEXTNEXT = properties.getProperty(GCUBE_DEVNEXT_NEXTNEXT_VARNAME);
@ -68,30 +61,29 @@ public class ScopedTest {
SA = properties.getProperty(SA_VARNAME); SA = properties.getProperty(SA_VARNAME);
DEFAULT_TEST_SCOPE = SA; DEFAULT_TEST_SCOPE = GCUBE_DEVNEXT;
ALTERNATIVE_TEST_SCOPE = GCUBE_DEVSEC; ALTERNATIVE_TEST_SCOPE = GCUBE_DEVSEC;
} }
public static String getCurrentScope(String token) throws ObjectNotFound, Exception{ public static String getCurrentScope(String token) throws ObjectNotFound, Exception {
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
String context = authorizationEntry.getContext(); String context = authorizationEntry.getContext();
logger.info("Context of token {} is {}", token, context); logger.info("Context of token {} is {}", token, context);
return context; return context;
} }
public static void setContext(String token) throws ObjectNotFound, Exception {
public static void setContext(String token) throws ObjectNotFound, Exception{
SecurityTokenProvider.instance.set(token); SecurityTokenProvider.instance.set(token);
ScopeProvider.instance.set(getCurrentScope(token)); ScopeProvider.instance.set(getCurrentScope(token));
} }
@BeforeClass @BeforeClass
public static void beforeClass() throws Exception{ public static void beforeClass() throws Exception {
setContext(DEFAULT_TEST_SCOPE); setContext(DEFAULT_TEST_SCOPE);
} }
@AfterClass @AfterClass
public static void afterClass() throws Exception{ public static void afterClass() throws Exception {
SecurityTokenProvider.instance.reset(); SecurityTokenProvider.instance.reset();
ScopeProvider.instance.reset(); ScopeProvider.instance.reset();
} }