From a109ffb5b3f3af90588e4ff38f131d800e1cfcc5 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Fri, 1 Mar 2019 15:48:43 +0000 Subject: [PATCH] Added utility git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/private/luca.frosini/infrastructure-tests@178438 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../java/org/gcube/phd/ContextStatistic.java | 38 ++-- src/test/java/org/gcube/phd/ResourceInfo.java | 21 ++ .../org/gcube/phd/ScopeBeanComparator.java | 13 ++ .../org/gcube/phd/StatisticsCollector.java | 76 +++++++ .../org/gcube/phd/StatisticsGenerator.java | 211 ++++++++++++++++++ 5 files changed, 344 insertions(+), 15 deletions(-) create mode 100644 src/test/java/org/gcube/phd/ScopeBeanComparator.java create mode 100644 src/test/java/org/gcube/phd/StatisticsCollector.java create mode 100644 src/test/java/org/gcube/phd/StatisticsGenerator.java diff --git a/src/test/java/org/gcube/phd/ContextStatistic.java b/src/test/java/org/gcube/phd/ContextStatistic.java index 8634ba9..098a38a 100644 --- a/src/test/java/org/gcube/phd/ContextStatistic.java +++ b/src/test/java/org/gcube/phd/ContextStatistic.java @@ -1,5 +1,6 @@ package org.gcube.phd; +import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -25,9 +26,9 @@ public class ContextStatistic { public ScopeBean getScopeBean() { return scopeBean; } - - private Map, Set>> resourceMap; - private Map, SummaryStatistics> statisticsMap; + + private Map,Set>> resourceMap; + private Map,SummaryStatistics> statisticsMap; public ContextStatistic(ScopeBean scopeBean) { this.scopeBean = scopeBean; @@ -45,8 +46,8 @@ public class ContextStatistic { } } - public Map, Set>> getAllResources() throws Exception { - if(resourceMap ==null) { + public Map,Set>> getAllResources() throws Exception { + if(resourceMap == null) { resourceMap = new HashMap<>(); List> classes = new ArrayList<>(); classes.add(GCoreEndpoint.class); @@ -62,19 +63,26 @@ public class ContextStatistic { return resourceMap; } - public Map, SummaryStatistics> getStatistcs() throws Exception { - if(statisticsMap==null) { - statisticsMap = new HashMap<>(); - getAllResources(); - for(Class clz : resourceMap.keySet()) { - SummaryStatistics summaryStatistics = new SummaryStatistics(); - Set> resources = resourceMap.get(clz); - for(ResourceInfo resourceInfo : resources) { - summaryStatistics.addValue(resourceInfo.getSize()); - } + public static Map,SummaryStatistics> generateStatistics( + Map,Set>> resourceMap) + throws UnsupportedEncodingException { + Map,SummaryStatistics> statisticsMap = new HashMap<>(); + for(Class clz : resourceMap.keySet()) { + SummaryStatistics summaryStatistics = new SummaryStatistics(); + Set> resources = resourceMap.get(clz); + for(ResourceInfo resourceInfo : resources) { + summaryStatistics.addValue(resourceInfo.getSize()); } } return statisticsMap; } + public Map,SummaryStatistics> getStatistcs() throws Exception { + if(statisticsMap == null) { + getAllResources(); + statisticsMap = generateStatistics(resourceMap); + } + return statisticsMap; + } + } diff --git a/src/test/java/org/gcube/phd/ResourceInfo.java b/src/test/java/org/gcube/phd/ResourceInfo.java index fb599a0..9c3bd51 100644 --- a/src/test/java/org/gcube/phd/ResourceInfo.java +++ b/src/test/java/org/gcube/phd/ResourceInfo.java @@ -34,4 +34,25 @@ public class ResourceInfo { return size; } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + @SuppressWarnings("unchecked") + ResourceInfo other = (ResourceInfo) obj; + return this.r.id().compareTo(other.r.id())==0; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * r.id().hashCode(); + return result; + } + } diff --git a/src/test/java/org/gcube/phd/ScopeBeanComparator.java b/src/test/java/org/gcube/phd/ScopeBeanComparator.java new file mode 100644 index 0000000..e0d0fc6 --- /dev/null +++ b/src/test/java/org/gcube/phd/ScopeBeanComparator.java @@ -0,0 +1,13 @@ +package org.gcube.phd; + +import java.util.Comparator; + +import org.gcube.common.scope.impl.ScopeBean; + +public class ScopeBeanComparator implements Comparator { + + @Override + public int compare(ScopeBean scopeBean1, ScopeBean scopeBean2) { + return scopeBean1.toString().compareTo(scopeBean2.toString()); + } +} diff --git a/src/test/java/org/gcube/phd/StatisticsCollector.java b/src/test/java/org/gcube/phd/StatisticsCollector.java new file mode 100644 index 0000000..268ce38 --- /dev/null +++ b/src/test/java/org/gcube/phd/StatisticsCollector.java @@ -0,0 +1,76 @@ +package org.gcube.phd; + +import static org.gcube.common.authorization.client.Constants.authorizationService; + +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.gcube.common.authorization.client.Constants; +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.authorization.library.provider.UserInfo; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.context.ContextElaborator; + +public class StatisticsCollector extends ContextElaborator { + + private Map contextCount; + private SortedMap sortedMap; + + public StatisticsCollector() {} + + @SuppressWarnings("unused") + private String generateUserToken(String context) throws Exception { + AuthorizationEntry authorizationEntry = Constants.authorizationService() + .get(SecurityTokenProvider.instance.get()); + UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo(); + String userToken = authorizationService().generateUserToken(userInfo, context); + logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken); + return userToken; + } + + + @Override + public void elaborateContext(ScopeBean scopeBean) throws Exception { + Type type = scopeBean.type(); + Integer integer = contextCount.get(type); + if(integer == null) { + integer = 0; + } + contextCount.put(type, integer + 1); + + + String contextFullName = scopeBean.toString(); + /* + String token = generateUserToken(contextFullName); + ContextTest.setContext(token); + */ + ScopeProvider.instance.set(contextFullName); + + ContextStatistic contextStatistic = new ContextStatistic(scopeBean); + sortedMap.put(scopeBean, contextStatistic); + } + + public SortedMap getStatistics() throws Exception{ + if(sortedMap == null) { + contextCount = new HashMap<>(); + sortedMap = new TreeMap<>(); + all(); + } + return sortedMap; + } + + public Map getContextCount() throws Exception { + if(contextCount == null) { + contextCount = new HashMap<>(); + sortedMap = new TreeMap<>(new ScopeBeanComparator()); + all(); + } + return contextCount; + } + +} diff --git a/src/test/java/org/gcube/phd/StatisticsGenerator.java b/src/test/java/org/gcube/phd/StatisticsGenerator.java new file mode 100644 index 0000000..b0905e4 --- /dev/null +++ b/src/test/java/org/gcube/phd/StatisticsGenerator.java @@ -0,0 +1,211 @@ +package org.gcube.phd; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.gcube.common.resources.gcore.Resource; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.testutility.ContextTest; +import org.junit.Test; + +public class StatisticsGenerator { + + private File directory; + private File statisticsFile; + + public StatisticsGenerator() { + directory = new File("/home/lucafrosini/Desktop/GlobalStatistics"); + if(!directory.exists()) { + directory.mkdirs(); + } + } + + private void printLineToFile(String line, File file) throws IOException { + synchronized(file) { + try(FileWriter fw = new FileWriter(file, true); + BufferedWriter bw = new BufferedWriter(fw); + PrintWriter out = new PrintWriter(bw)) { + out.println(line); + out.flush(); + } catch(IOException e) { + throw e; + } + } + } + + private void addContextCount(Map contextCount) throws Exception { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("Context Type"); + stringBuffer.append(","); + stringBuffer.append("Count"); + printLineToFile(stringBuffer.toString(), statisticsFile); + + for(Type type : contextCount.keySet()) { + stringBuffer = new StringBuffer(); + stringBuffer.append(type.name()); + stringBuffer.append(","); + stringBuffer.append(contextCount.get(type)); + printLineToFile(stringBuffer.toString(), statisticsFile); + } + + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + } + + private void addHeader() throws IOException { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("Resource Type"); + stringBuffer.append(","); + stringBuffer.append("Count"); + stringBuffer.append(","); + stringBuffer.append("Avg Size"); + stringBuffer.append(","); + stringBuffer.append("Max"); + stringBuffer.append(","); + stringBuffer.append("Min"); + stringBuffer.append(","); + stringBuffer.append("Variance"); + printLineToFile(stringBuffer.toString(), statisticsFile); + } + + private void addResourceStatistic(Class clz, SummaryStatistics summaryStatistics) + throws IOException { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append(clz.getSimpleName()); + stringBuffer.append(","); + stringBuffer.append(summaryStatistics.getN()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getMean()).intValue()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getMax()).intValue()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getVariance()).intValue()); + printLineToFile(stringBuffer.toString(), statisticsFile); + } + + private void addStatistics(Map,SummaryStatistics> statistics) throws Exception { + for(Class clz : statistics.keySet()) { + addResourceStatistic(clz, statistics.get(clz)); + } + } + + private void generateStatistics( + SortedMap,Set>>> aggregatedResources) + throws Exception { + + for(ScopeBean scopeBean : aggregatedResources.keySet()) { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append(scopeBean.toString()); + stringBuffer.append(" Statistics"); + printLineToFile(stringBuffer.toString(), statisticsFile); + addHeader(); + + Map,Set>> resourceMap = aggregatedResources + .get(scopeBean); + Map,SummaryStatistics> statistics = ContextStatistic + .generateStatistics(resourceMap); + addStatistics(statistics); + + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + } + } + + private SortedMap,Set>>> getAggregatedResources( + SortedMap contextStatistics) throws Exception { + + SortedMap,Set>>> aggregatedResources = new TreeMap<>( + new ScopeBeanComparator()); + + Map,Set>> infrastructureResourcesByClass = null; + Map,Set>> voResourcesByClass = null; + + for(ScopeBean scopeBean : contextStatistics.keySet()) { + + ContextStatistic contextStatistic = contextStatistics.get(scopeBean); + Map,Set>> resourceMap = contextStatistic + .getAllResources(); + + Type type = scopeBean.type(); + switch(type) { + case INFRASTRUCTURE: + infrastructureResourcesByClass = new HashMap<>(); + aggregatedResources.put(scopeBean, infrastructureResourcesByClass); + break; + + case VO: + voResourcesByClass = new HashMap<>(); + aggregatedResources.put(scopeBean, voResourcesByClass); + break; + + case VRE: + break; + + default: + break; + } + + for(Class clz : resourceMap.keySet()) { + Set> resources = resourceMap.get(clz); + + Set> infrastructureResources = infrastructureResourcesByClass.get(clz); + if(infrastructureResources == null) { + infrastructureResources = new HashSet<>(); + infrastructureResourcesByClass.put(clz, infrastructureResources); + } + + infrastructureResources.addAll(resources); + + Set> voResources = voResourcesByClass.get(clz); + if(voResources == null) { + voResources = new HashSet<>(); + voResourcesByClass.put(clz, voResources); + } + voResources.addAll(resources); + + } + + } + + return aggregatedResources; + + } + + private void generateStatistics() throws Exception { + statisticsFile = new File(directory, "all.csv"); + if(statisticsFile.exists()) { + statisticsFile.delete(); + } + statisticsFile.createNewFile(); + StatisticsCollector statisticsCollector = new StatisticsCollector(); + Map contextCount = statisticsCollector.getContextCount(); + addContextCount(contextCount); + + SortedMap contextStatistics = statisticsCollector.getStatistics(); + SortedMap,Set>>> aggregatedResources = getAggregatedResources( + contextStatistics); + generateStatistics(aggregatedResources); + } + + @Test + public void test() throws Exception { + ContextTest.setContextByName("/d4science.research-infrastructures.eu"); + generateStatistics(); + } + +}