Added utility
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/private/luca.frosini/infrastructure-tests@178459 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
a109ffb5b3
commit
5d8abaedea
|
|
@ -1,26 +1,60 @@
|
|||
package org.gcube.phd;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
|
||||
import org.gcube.accounting.aggregator.utility.Utility;
|
||||
import org.gcube.accounting.analytics.Filter;
|
||||
import org.gcube.accounting.analytics.Info;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
import org.gcube.common.authorization.client.Constants;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.authorization.library.provider.UserInfo;
|
||||
import org.gcube.common.resources.gcore.GCoreEndpoint;
|
||||
import org.gcube.common.resources.gcore.GenericResource;
|
||||
import org.gcube.common.resources.gcore.HostingNode;
|
||||
import org.gcube.common.resources.gcore.Resource;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||
import org.gcube.common.resources.gcore.Software;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.common.scope.impl.ScopeBean;
|
||||
import org.gcube.documentstore.records.AggregatedRecord;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
import org.gcube.resources.discovery.icclient.ICFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class ContextStatistic {
|
||||
|
||||
protected Logger logger = LoggerFactory.getLogger(ContextStatistic.class);
|
||||
|
||||
public static Calendar START_TIME;
|
||||
public static Calendar END_TIME;
|
||||
private static AccountingPersistenceQuery apq;
|
||||
|
||||
static {
|
||||
START_TIME = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1);
|
||||
END_TIME = Utility.getAggregationStartCalendar(2017, Calendar.DECEMBER, 1);
|
||||
apq = AccountingPersistenceQueryFactory.getInstance();
|
||||
}
|
||||
|
||||
private ScopeBean scopeBean;
|
||||
|
||||
public ScopeBean getScopeBean() {
|
||||
|
|
@ -29,6 +63,7 @@ public class ContextStatistic {
|
|||
|
||||
private Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap;
|
||||
private Map<Class<? extends Resource>,SummaryStatistics> statisticsMap;
|
||||
private SortedMap<Calendar, Info> accountingData;
|
||||
|
||||
public ContextStatistic(ScopeBean scopeBean) {
|
||||
this.scopeBean = scopeBean;
|
||||
|
|
@ -36,9 +71,13 @@ public class ContextStatistic {
|
|||
|
||||
public <R extends Resource> void analizeInstances(Class<R> clz) throws Exception {
|
||||
Set<ResourceInfo<? extends Resource>> resources = resourceMap.get(clz);
|
||||
|
||||
if(resources==null) {
|
||||
resources = new TreeSet<>();
|
||||
resourceMap.put(clz, resources);
|
||||
}
|
||||
DiscoveryClient<R> client = ICFactory.clientFor(clz);
|
||||
SimpleQuery query = ICFactory.queryFor(clz);
|
||||
// logger.debug(query.expression());
|
||||
List<R> instances = client.submit(query);
|
||||
for(R r : instances) {
|
||||
ResourceInfo<R> resourceInfo = new ResourceInfo<R>(r);
|
||||
|
|
@ -46,6 +85,17 @@ public class ContextStatistic {
|
|||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private String generateUserToken(String context) throws Exception {
|
||||
AuthorizationEntry authorizationEntry = Constants.authorizationService()
|
||||
.get(SecurityTokenProvider.instance.get());
|
||||
UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo();
|
||||
String userToken = authorizationService().generateUserToken(userInfo, context);
|
||||
logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken);
|
||||
return userToken;
|
||||
}
|
||||
|
||||
|
||||
public Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> getAllResources() throws Exception {
|
||||
if(resourceMap == null) {
|
||||
resourceMap = new HashMap<>();
|
||||
|
|
@ -56,7 +106,15 @@ public class ContextStatistic {
|
|||
classes.add(GenericResource.class);
|
||||
classes.add(Software.class);
|
||||
|
||||
String contextFullName = scopeBean.toString();
|
||||
/*
|
||||
String token = generateUserToken(contextFullName);
|
||||
ContextTest.setContext(token);
|
||||
*/
|
||||
ScopeProvider.instance.set(contextFullName);
|
||||
|
||||
for(Class<? extends Resource> clz : classes) {
|
||||
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
|
||||
analizeInstances(clz);
|
||||
}
|
||||
}
|
||||
|
|
@ -73,11 +131,32 @@ public class ContextStatistic {
|
|||
for(ResourceInfo<? extends Resource> resourceInfo : resources) {
|
||||
summaryStatistics.addValue(resourceInfo.getSize());
|
||||
}
|
||||
statisticsMap.put(clz, summaryStatistics);
|
||||
}
|
||||
return statisticsMap;
|
||||
}
|
||||
|
||||
public Map<Class<? extends Resource>,SummaryStatistics> getStatistcs() throws Exception {
|
||||
public SortedMap<Calendar, Info> getAccountingData() throws Exception {
|
||||
if(accountingData==null) {
|
||||
|
||||
TemporalConstraint temporalConstraint = new TemporalConstraint(
|
||||
START_TIME.getTimeInMillis(), END_TIME.getTimeInMillis(),
|
||||
AggregationMode.MONTHLY);
|
||||
List<Filter> filters = new ArrayList<>();
|
||||
Filter filter = new Filter("serviceClass", "InformationSystem");
|
||||
filters.add(filter);
|
||||
|
||||
Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
|
||||
|
||||
// AccountingPersistenceQueryFactory.getForcedQueryScope().set(scopeBean.toString());
|
||||
|
||||
accountingData = apq.getTimeSeries(clz, temporalConstraint,
|
||||
filters, true);
|
||||
}
|
||||
return accountingData;
|
||||
}
|
||||
|
||||
public Map<Class<? extends Resource>,SummaryStatistics> getStatistics() throws Exception {
|
||||
if(statisticsMap == null) {
|
||||
getAllResources();
|
||||
statisticsMap = generateStatistics(resourceMap);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import java.io.UnsupportedEncodingException;
|
|||
import org.gcube.common.resources.gcore.Resource;
|
||||
import org.gcube.common.resources.gcore.Resources;
|
||||
|
||||
public class ResourceInfo<R extends Resource> {
|
||||
public class ResourceInfo<R extends Resource> implements Comparable<ResourceInfo<R>> {
|
||||
|
||||
private R r;
|
||||
private String unmarshalledResource;
|
||||
|
|
@ -25,6 +25,10 @@ public class ResourceInfo<R extends Resource> {
|
|||
return unmarshalledResource;
|
||||
}
|
||||
|
||||
public String getResourceID() {
|
||||
return r.id();
|
||||
}
|
||||
|
||||
public int getSize() throws UnsupportedEncodingException {
|
||||
if(size==null) {
|
||||
String unmarshalledR = getUnmarshalledResource();
|
||||
|
|
@ -44,7 +48,7 @@ public class ResourceInfo<R extends Resource> {
|
|||
return false;
|
||||
@SuppressWarnings("unchecked")
|
||||
ResourceInfo<R> other = (ResourceInfo<R>) obj;
|
||||
return this.r.id().compareTo(other.r.id())==0;
|
||||
return this.compareTo(other)==0;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -54,5 +58,10 @@ public class ResourceInfo<R extends Resource> {
|
|||
result = prime * r.id().hashCode();
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(ResourceInfo<R> other) {
|
||||
return this.r.id().compareTo(other.r.id());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,10 @@
|
|||
package org.gcube.phd;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.gcube.common.authorization.client.Constants;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.authorization.library.provider.UserInfo;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.common.scope.impl.ScopeBean;
|
||||
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||
import org.gcube.context.ContextElaborator;
|
||||
|
|
@ -23,17 +16,6 @@ public class StatisticsCollector extends ContextElaborator {
|
|||
|
||||
public StatisticsCollector() {}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private String generateUserToken(String context) throws Exception {
|
||||
AuthorizationEntry authorizationEntry = Constants.authorizationService()
|
||||
.get(SecurityTokenProvider.instance.get());
|
||||
UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo();
|
||||
String userToken = authorizationService().generateUserToken(userInfo, context);
|
||||
logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken);
|
||||
return userToken;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void elaborateContext(ScopeBean scopeBean) throws Exception {
|
||||
Type type = scopeBean.type();
|
||||
|
|
@ -43,14 +25,6 @@ public class StatisticsCollector extends ContextElaborator {
|
|||
}
|
||||
contextCount.put(type, integer + 1);
|
||||
|
||||
|
||||
String contextFullName = scopeBean.toString();
|
||||
/*
|
||||
String token = generateUserToken(contextFullName);
|
||||
ContextTest.setContext(token);
|
||||
*/
|
||||
ScopeProvider.instance.set(contextFullName);
|
||||
|
||||
ContextStatistic contextStatistic = new ContextStatistic(scopeBean);
|
||||
sortedMap.put(scopeBean, contextStatistic);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import java.io.File;
|
|||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Calendar;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
|
|
@ -13,19 +15,25 @@ import java.util.SortedMap;
|
|||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
|
||||
import org.gcube.accounting.analytics.Info;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||
import org.gcube.common.resources.gcore.Resource;
|
||||
import org.gcube.common.scope.impl.ScopeBean;
|
||||
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||
import org.gcube.testutility.ContextTest;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class StatisticsGenerator {
|
||||
|
||||
protected Logger logger = LoggerFactory.getLogger(this.getClass());
|
||||
|
||||
private File directory;
|
||||
private File statisticsFile;
|
||||
|
||||
public StatisticsGenerator() {
|
||||
directory = new File("/home/lucafrosini/Desktop/GlobalStatistics");
|
||||
directory = new File("/home/lucafrosini/Dropbox/GlobalStatistics");
|
||||
if(!directory.exists()) {
|
||||
directory.mkdirs();
|
||||
}
|
||||
|
|
@ -51,7 +59,7 @@ public class StatisticsGenerator {
|
|||
stringBuffer.append("Count");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
|
||||
for(Type type : contextCount.keySet()) {
|
||||
for(Type type : Type.values()) {
|
||||
stringBuffer = new StringBuffer();
|
||||
stringBuffer.append(type.name());
|
||||
stringBuffer.append(",");
|
||||
|
|
@ -77,7 +85,10 @@ public class StatisticsGenerator {
|
|||
stringBuffer.append(",");
|
||||
stringBuffer.append("Min");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append("Variance");
|
||||
stringBuffer.append("Standard Deviation");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append("Geometric Mean");
|
||||
stringBuffer.append(",");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
}
|
||||
|
||||
|
|
@ -94,7 +105,10 @@ public class StatisticsGenerator {
|
|||
stringBuffer.append(",");
|
||||
stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue());
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(Double.valueOf(summaryStatistics.getVariance()).intValue());
|
||||
stringBuffer.append(Double.valueOf(summaryStatistics.getStandardDeviation()).intValue());
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(Double.valueOf(summaryStatistics.getGeometricMean()).intValue());
|
||||
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
}
|
||||
|
||||
|
|
@ -104,28 +118,73 @@ public class StatisticsGenerator {
|
|||
}
|
||||
}
|
||||
|
||||
private void generateStatistics(
|
||||
private void contextStatisticReport(ScopeBean scopeBean, Map<Class<? extends Resource>,SummaryStatistics> statistics) throws Exception {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append(scopeBean.toString());
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
addHeader();
|
||||
|
||||
addStatistics(statistics);
|
||||
}
|
||||
|
||||
private void contextAccountingReport(ScopeBean scopeBean, SortedMap<Calendar,Info> accountingData) throws Exception {
|
||||
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
simpleDateFormat.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
||||
StringBuffer calendarBuffer = new StringBuffer();
|
||||
StringBuffer dataBuffer = new StringBuffer();
|
||||
boolean first = true;
|
||||
for(Calendar calendar : accountingData.keySet()) {
|
||||
if(first) {
|
||||
first = false;
|
||||
}else {
|
||||
calendarBuffer.append(",");
|
||||
dataBuffer.append(",");
|
||||
}
|
||||
calendarBuffer.append(simpleDateFormat.format(calendar.getTime()));
|
||||
Info info = accountingData.get(calendar);
|
||||
int operationCount = (int) info.getValue().get("operationCount");
|
||||
dataBuffer.append(operationCount);
|
||||
}
|
||||
printLineToFile(calendarBuffer.toString(), statisticsFile);
|
||||
printLineToFile(dataBuffer.toString(), statisticsFile);
|
||||
}
|
||||
|
||||
private void generateAggregatedStatistics(
|
||||
SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources)
|
||||
throws Exception {
|
||||
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("######################################", statisticsFile);
|
||||
printLineToFile("Aggregated Statistics", statisticsFile);
|
||||
|
||||
for(ScopeBean scopeBean : aggregatedResources.keySet()) {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append(scopeBean.toString());
|
||||
stringBuffer.append(" Statistics");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
addHeader();
|
||||
|
||||
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap = aggregatedResources
|
||||
.get(scopeBean);
|
||||
Map<Class<? extends Resource>,SummaryStatistics> statistics = ContextStatistic
|
||||
.generateStatistics(resourceMap);
|
||||
addStatistics(statistics);
|
||||
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
contextStatisticReport(scopeBean, statistics);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void generateContextStatisticReport(ScopeBean scopeBean, ContextStatistic contextStatistic) throws Exception {
|
||||
contextStatisticReport(scopeBean, contextStatistic.getStatistics());
|
||||
/*
|
||||
try {
|
||||
contextAccountingReport(scopeBean, contextStatistic.getAccountingData());
|
||||
}catch (Exception e) {
|
||||
logger.error("Unable to Generate AccountignData resport for contes {}", scopeBean.toString(), e);
|
||||
printLineToFile("PARTIAL OR INVALID ACCOUNTING DATA", statisticsFile);
|
||||
}
|
||||
*/
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
}
|
||||
|
||||
private SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> getAggregatedResources(
|
||||
SortedMap<ScopeBean,ContextStatistic> contextStatistics) throws Exception {
|
||||
|
||||
|
|
@ -138,6 +197,11 @@ public class StatisticsGenerator {
|
|||
for(ScopeBean scopeBean : contextStatistics.keySet()) {
|
||||
|
||||
ContextStatistic contextStatistic = contextStatistics.get(scopeBean);
|
||||
generateContextStatisticReport(scopeBean, contextStatistic);
|
||||
|
||||
|
||||
logger.debug("Generating AggregatedStatistics for {}", scopeBean.toString());
|
||||
|
||||
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap = contextStatistic
|
||||
.getAllResources();
|
||||
|
||||
|
|
@ -160,23 +224,45 @@ public class StatisticsGenerator {
|
|||
break;
|
||||
}
|
||||
|
||||
File dir = new File(directory, scopeBean.toString());
|
||||
if(!dir.exists()) {
|
||||
dir.mkdirs();
|
||||
}
|
||||
|
||||
for(Class<? extends Resource> clz : resourceMap.keySet()) {
|
||||
Set<ResourceInfo<? extends Resource>> resources = resourceMap.get(clz);
|
||||
/*
|
||||
File resourceDir = new File(dir, "__Resources");
|
||||
resourceDir = new File(resourceDir, clz.getSimpleName());
|
||||
if(!resourceDir.exists()) {
|
||||
resourceDir.mkdirs();
|
||||
}
|
||||
for(ResourceInfo<? extends Resource> resourceInfo : resources) {
|
||||
File f = new File(resourceDir, resourceInfo.getResourceID());
|
||||
if(f.exists()) {
|
||||
f.delete();
|
||||
}
|
||||
f.createNewFile();
|
||||
printLineToFile(resourceInfo.getUnmarshalledResource(),f);
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
Set<ResourceInfo<? extends Resource>> infrastructureResources = infrastructureResourcesByClass.get(clz);
|
||||
if(infrastructureResources == null) {
|
||||
infrastructureResources = new HashSet<>();
|
||||
infrastructureResourcesByClass.put(clz, infrastructureResources);
|
||||
}
|
||||
|
||||
infrastructureResources.addAll(resources);
|
||||
|
||||
Set<ResourceInfo<? extends Resource>> voResources = voResourcesByClass.get(clz);
|
||||
if(voResources == null) {
|
||||
voResources = new HashSet<>();
|
||||
voResourcesByClass.put(clz, voResources);
|
||||
if(type != Type.INFRASTRUCTURE) {
|
||||
Set<ResourceInfo<? extends Resource>> voResources = voResourcesByClass.get(clz);
|
||||
if(voResources == null) {
|
||||
voResources = new HashSet<>();
|
||||
voResourcesByClass.put(clz, voResources);
|
||||
}
|
||||
voResources.addAll(resources);
|
||||
}
|
||||
voResources.addAll(resources);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -186,12 +272,15 @@ public class StatisticsGenerator {
|
|||
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void generateStatistics() throws Exception {
|
||||
statisticsFile = new File(directory, "all.csv");
|
||||
statisticsFile = new File(directory, "all-corrected.csv");
|
||||
if(statisticsFile.exists()) {
|
||||
statisticsFile.delete();
|
||||
}
|
||||
statisticsFile.createNewFile();
|
||||
|
||||
StatisticsCollector statisticsCollector = new StatisticsCollector();
|
||||
Map<Type,Integer> contextCount = statisticsCollector.getContextCount();
|
||||
addContextCount(contextCount);
|
||||
|
|
@ -199,7 +288,7 @@ public class StatisticsGenerator {
|
|||
SortedMap<ScopeBean,ContextStatistic> contextStatistics = statisticsCollector.getStatistics();
|
||||
SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources = getAggregatedResources(
|
||||
contextStatistics);
|
||||
generateStatistics(aggregatedResources);
|
||||
generateAggregatedStatistics(aggregatedResources);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
Loading…
Reference in New Issue