Added utility
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/private/luca.frosini/infrastructure-tests@178438 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
bf4229014c
commit
a109ffb5b3
|
@ -1,5 +1,6 @@
|
||||||
package org.gcube.phd;
|
package org.gcube.phd;
|
||||||
|
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -25,9 +26,9 @@ public class ContextStatistic {
|
||||||
public ScopeBean getScopeBean() {
|
public ScopeBean getScopeBean() {
|
||||||
return scopeBean;
|
return scopeBean;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<Class<? extends Resource>, Set<ResourceInfo<? extends Resource>>> resourceMap;
|
private Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap;
|
||||||
private Map<Class<? extends Resource>, SummaryStatistics> statisticsMap;
|
private Map<Class<? extends Resource>,SummaryStatistics> statisticsMap;
|
||||||
|
|
||||||
public ContextStatistic(ScopeBean scopeBean) {
|
public ContextStatistic(ScopeBean scopeBean) {
|
||||||
this.scopeBean = scopeBean;
|
this.scopeBean = scopeBean;
|
||||||
|
@ -45,8 +46,8 @@ public class ContextStatistic {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<Class<? extends Resource>, Set<ResourceInfo<? extends Resource>>> getAllResources() throws Exception {
|
public Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> getAllResources() throws Exception {
|
||||||
if(resourceMap ==null) {
|
if(resourceMap == null) {
|
||||||
resourceMap = new HashMap<>();
|
resourceMap = new HashMap<>();
|
||||||
List<Class<? extends Resource>> classes = new ArrayList<>();
|
List<Class<? extends Resource>> classes = new ArrayList<>();
|
||||||
classes.add(GCoreEndpoint.class);
|
classes.add(GCoreEndpoint.class);
|
||||||
|
@ -62,19 +63,26 @@ public class ContextStatistic {
|
||||||
return resourceMap;
|
return resourceMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<Class<? extends Resource>, SummaryStatistics> getStatistcs() throws Exception {
|
public static Map<Class<? extends Resource>,SummaryStatistics> generateStatistics(
|
||||||
if(statisticsMap==null) {
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap)
|
||||||
statisticsMap = new HashMap<>();
|
throws UnsupportedEncodingException {
|
||||||
getAllResources();
|
Map<Class<? extends Resource>,SummaryStatistics> statisticsMap = new HashMap<>();
|
||||||
for(Class<? extends Resource> clz : resourceMap.keySet()) {
|
for(Class<? extends Resource> clz : resourceMap.keySet()) {
|
||||||
SummaryStatistics summaryStatistics = new SummaryStatistics();
|
SummaryStatistics summaryStatistics = new SummaryStatistics();
|
||||||
Set<ResourceInfo<? extends Resource>> resources = resourceMap.get(clz);
|
Set<ResourceInfo<? extends Resource>> resources = resourceMap.get(clz);
|
||||||
for(ResourceInfo<? extends Resource> resourceInfo : resources) {
|
for(ResourceInfo<? extends Resource> resourceInfo : resources) {
|
||||||
summaryStatistics.addValue(resourceInfo.getSize());
|
summaryStatistics.addValue(resourceInfo.getSize());
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return statisticsMap;
|
return statisticsMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<Class<? extends Resource>,SummaryStatistics> getStatistcs() throws Exception {
|
||||||
|
if(statisticsMap == null) {
|
||||||
|
getAllResources();
|
||||||
|
statisticsMap = generateStatistics(resourceMap);
|
||||||
|
}
|
||||||
|
return statisticsMap;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,4 +34,25 @@ public class ResourceInfo<R extends Resource> {
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object obj) {
|
||||||
|
if (this == obj)
|
||||||
|
return true;
|
||||||
|
if (obj == null)
|
||||||
|
return false;
|
||||||
|
if (getClass() != obj.getClass())
|
||||||
|
return false;
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
ResourceInfo<R> other = (ResourceInfo<R>) obj;
|
||||||
|
return this.r.id().compareTo(other.r.id())==0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
final int prime = 31;
|
||||||
|
int result = 1;
|
||||||
|
result = prime * r.id().hashCode();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
package org.gcube.phd;
|
||||||
|
|
||||||
|
import java.util.Comparator;
|
||||||
|
|
||||||
|
import org.gcube.common.scope.impl.ScopeBean;
|
||||||
|
|
||||||
|
public class ScopeBeanComparator implements Comparator<ScopeBean> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int compare(ScopeBean scopeBean1, ScopeBean scopeBean2) {
|
||||||
|
return scopeBean1.toString().compareTo(scopeBean2.toString());
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,76 @@
|
||||||
|
package org.gcube.phd;
|
||||||
|
|
||||||
|
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.SortedMap;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
|
||||||
|
import org.gcube.common.authorization.client.Constants;
|
||||||
|
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||||
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.common.authorization.library.provider.UserInfo;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.gcube.common.scope.impl.ScopeBean;
|
||||||
|
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||||
|
import org.gcube.context.ContextElaborator;
|
||||||
|
|
||||||
|
public class StatisticsCollector extends ContextElaborator {
|
||||||
|
|
||||||
|
private Map<Type,Integer> contextCount;
|
||||||
|
private SortedMap<ScopeBean, ContextStatistic> sortedMap;
|
||||||
|
|
||||||
|
public StatisticsCollector() {}
|
||||||
|
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
private String generateUserToken(String context) throws Exception {
|
||||||
|
AuthorizationEntry authorizationEntry = Constants.authorizationService()
|
||||||
|
.get(SecurityTokenProvider.instance.get());
|
||||||
|
UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo();
|
||||||
|
String userToken = authorizationService().generateUserToken(userInfo, context);
|
||||||
|
logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken);
|
||||||
|
return userToken;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void elaborateContext(ScopeBean scopeBean) throws Exception {
|
||||||
|
Type type = scopeBean.type();
|
||||||
|
Integer integer = contextCount.get(type);
|
||||||
|
if(integer == null) {
|
||||||
|
integer = 0;
|
||||||
|
}
|
||||||
|
contextCount.put(type, integer + 1);
|
||||||
|
|
||||||
|
|
||||||
|
String contextFullName = scopeBean.toString();
|
||||||
|
/*
|
||||||
|
String token = generateUserToken(contextFullName);
|
||||||
|
ContextTest.setContext(token);
|
||||||
|
*/
|
||||||
|
ScopeProvider.instance.set(contextFullName);
|
||||||
|
|
||||||
|
ContextStatistic contextStatistic = new ContextStatistic(scopeBean);
|
||||||
|
sortedMap.put(scopeBean, contextStatistic);
|
||||||
|
}
|
||||||
|
|
||||||
|
public SortedMap<ScopeBean, ContextStatistic> getStatistics() throws Exception{
|
||||||
|
if(sortedMap == null) {
|
||||||
|
contextCount = new HashMap<>();
|
||||||
|
sortedMap = new TreeMap<>();
|
||||||
|
all();
|
||||||
|
}
|
||||||
|
return sortedMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<Type,Integer> getContextCount() throws Exception {
|
||||||
|
if(contextCount == null) {
|
||||||
|
contextCount = new HashMap<>();
|
||||||
|
sortedMap = new TreeMap<>(new ScopeBeanComparator());
|
||||||
|
all();
|
||||||
|
}
|
||||||
|
return contextCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,211 @@
|
||||||
|
package org.gcube.phd;
|
||||||
|
|
||||||
|
import java.io.BufferedWriter;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileWriter;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.SortedMap;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
|
||||||
|
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
|
||||||
|
import org.gcube.common.resources.gcore.Resource;
|
||||||
|
import org.gcube.common.scope.impl.ScopeBean;
|
||||||
|
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||||
|
import org.gcube.testutility.ContextTest;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class StatisticsGenerator {
|
||||||
|
|
||||||
|
private File directory;
|
||||||
|
private File statisticsFile;
|
||||||
|
|
||||||
|
public StatisticsGenerator() {
|
||||||
|
directory = new File("/home/lucafrosini/Desktop/GlobalStatistics");
|
||||||
|
if(!directory.exists()) {
|
||||||
|
directory.mkdirs();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void printLineToFile(String line, File file) throws IOException {
|
||||||
|
synchronized(file) {
|
||||||
|
try(FileWriter fw = new FileWriter(file, true);
|
||||||
|
BufferedWriter bw = new BufferedWriter(fw);
|
||||||
|
PrintWriter out = new PrintWriter(bw)) {
|
||||||
|
out.println(line);
|
||||||
|
out.flush();
|
||||||
|
} catch(IOException e) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addContextCount(Map<Type,Integer> contextCount) throws Exception {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append("Context Type");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Count");
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
|
||||||
|
for(Type type : contextCount.keySet()) {
|
||||||
|
stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append(type.name());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(contextCount.get(type));
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addHeader() throws IOException {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append("Resource Type");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Count");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Avg Size");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Max");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Min");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Variance");
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addResourceStatistic(Class<? extends Resource> clz, SummaryStatistics summaryStatistics)
|
||||||
|
throws IOException {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append(clz.getSimpleName());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(summaryStatistics.getN());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMean()).intValue());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMax()).intValue());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(Double.valueOf(summaryStatistics.getVariance()).intValue());
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addStatistics(Map<Class<? extends Resource>,SummaryStatistics> statistics) throws Exception {
|
||||||
|
for(Class<? extends Resource> clz : statistics.keySet()) {
|
||||||
|
addResourceStatistic(clz, statistics.get(clz));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void generateStatistics(
|
||||||
|
SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources)
|
||||||
|
throws Exception {
|
||||||
|
|
||||||
|
for(ScopeBean scopeBean : aggregatedResources.keySet()) {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append(scopeBean.toString());
|
||||||
|
stringBuffer.append(" Statistics");
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
addHeader();
|
||||||
|
|
||||||
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap = aggregatedResources
|
||||||
|
.get(scopeBean);
|
||||||
|
Map<Class<? extends Resource>,SummaryStatistics> statistics = ContextStatistic
|
||||||
|
.generateStatistics(resourceMap);
|
||||||
|
addStatistics(statistics);
|
||||||
|
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> getAggregatedResources(
|
||||||
|
SortedMap<ScopeBean,ContextStatistic> contextStatistics) throws Exception {
|
||||||
|
|
||||||
|
SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources = new TreeMap<>(
|
||||||
|
new ScopeBeanComparator());
|
||||||
|
|
||||||
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> infrastructureResourcesByClass = null;
|
||||||
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> voResourcesByClass = null;
|
||||||
|
|
||||||
|
for(ScopeBean scopeBean : contextStatistics.keySet()) {
|
||||||
|
|
||||||
|
ContextStatistic contextStatistic = contextStatistics.get(scopeBean);
|
||||||
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap = contextStatistic
|
||||||
|
.getAllResources();
|
||||||
|
|
||||||
|
Type type = scopeBean.type();
|
||||||
|
switch(type) {
|
||||||
|
case INFRASTRUCTURE:
|
||||||
|
infrastructureResourcesByClass = new HashMap<>();
|
||||||
|
aggregatedResources.put(scopeBean, infrastructureResourcesByClass);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case VO:
|
||||||
|
voResourcesByClass = new HashMap<>();
|
||||||
|
aggregatedResources.put(scopeBean, voResourcesByClass);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case VRE:
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
for(Class<? extends Resource> clz : resourceMap.keySet()) {
|
||||||
|
Set<ResourceInfo<? extends Resource>> resources = resourceMap.get(clz);
|
||||||
|
|
||||||
|
Set<ResourceInfo<? extends Resource>> infrastructureResources = infrastructureResourcesByClass.get(clz);
|
||||||
|
if(infrastructureResources == null) {
|
||||||
|
infrastructureResources = new HashSet<>();
|
||||||
|
infrastructureResourcesByClass.put(clz, infrastructureResources);
|
||||||
|
}
|
||||||
|
|
||||||
|
infrastructureResources.addAll(resources);
|
||||||
|
|
||||||
|
Set<ResourceInfo<? extends Resource>> voResources = voResourcesByClass.get(clz);
|
||||||
|
if(voResources == null) {
|
||||||
|
voResources = new HashSet<>();
|
||||||
|
voResourcesByClass.put(clz, voResources);
|
||||||
|
}
|
||||||
|
voResources.addAll(resources);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
return aggregatedResources;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private void generateStatistics() throws Exception {
|
||||||
|
statisticsFile = new File(directory, "all.csv");
|
||||||
|
if(statisticsFile.exists()) {
|
||||||
|
statisticsFile.delete();
|
||||||
|
}
|
||||||
|
statisticsFile.createNewFile();
|
||||||
|
StatisticsCollector statisticsCollector = new StatisticsCollector();
|
||||||
|
Map<Type,Integer> contextCount = statisticsCollector.getContextCount();
|
||||||
|
addContextCount(contextCount);
|
||||||
|
|
||||||
|
SortedMap<ScopeBean,ContextStatistic> contextStatistics = statisticsCollector.getStatistics();
|
||||||
|
SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources = getAggregatedResources(
|
||||||
|
contextStatistics);
|
||||||
|
generateStatistics(aggregatedResources);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void test() throws Exception {
|
||||||
|
ContextTest.setContextByName("/d4science.research-infrastructures.eu");
|
||||||
|
generateStatistics();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue