From 8ec41a6d807771dd0bd026d3a79095d1d187acf3 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Fri, 1 Mar 2019 13:00:35 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/private/luca.frosini/infrastructure-tests@178411 82a268e6-3cf1-43bd-a215-b396298e98cf --- ...atistics.java => OLDGlobalStatistics.java} | 6 +- .../org/gcube/phd/StatisticsAggregator.java | 304 ++++++++++++++++++ 2 files changed, 307 insertions(+), 3 deletions(-) rename src/test/java/org/gcube/phd/{GlobalStatistics.java => OLDGlobalStatistics.java} (97%) create mode 100644 src/test/java/org/gcube/phd/StatisticsAggregator.java diff --git a/src/test/java/org/gcube/phd/GlobalStatistics.java b/src/test/java/org/gcube/phd/OLDGlobalStatistics.java similarity index 97% rename from src/test/java/org/gcube/phd/GlobalStatistics.java rename to src/test/java/org/gcube/phd/OLDGlobalStatistics.java index ee536e0..278f0f8 100644 --- a/src/test/java/org/gcube/phd/GlobalStatistics.java +++ b/src/test/java/org/gcube/phd/OLDGlobalStatistics.java @@ -32,13 +32,13 @@ import org.gcube.resources.discovery.icclient.ICFactory; import org.gcube.testutility.ContextTest; import org.junit.Test; -public class GlobalStatistics extends ContextElaborator { +public class OLDGlobalStatistics extends ContextElaborator { private File directory; private Set visited; - public GlobalStatistics() { + public OLDGlobalStatistics() { visited = new HashSet(); directory = new File("/home/lucafrosini/Desktop/Statistiche/Global"); if(!directory.exists()) { @@ -71,7 +71,7 @@ public class GlobalStatistics extends ContextElaborator { if(!statisticsFile.exists()) { statisticsFile.createNewFile(); stringBuffer.append(r.getClass().getSimpleName()); - stringBuffer.append("(byte size)"); + stringBuffer.append(" (byte)"); printLineToFile(stringBuffer.toString(), statisticsFile); stringBuffer = new StringBuffer(); } diff --git a/src/test/java/org/gcube/phd/StatisticsAggregator.java b/src/test/java/org/gcube/phd/StatisticsAggregator.java new file mode 100644 index 0000000..396a516 --- /dev/null +++ b/src/test/java/org/gcube/phd/StatisticsAggregator.java @@ -0,0 +1,304 @@ +package org.gcube.phd; + +import static org.gcube.common.authorization.client.Constants.authorizationService; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.gcube.common.authorization.client.Constants; +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.authorization.library.provider.UserInfo; +import org.gcube.common.resources.gcore.GCoreEndpoint; +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.HostingNode; +import org.gcube.common.resources.gcore.Resource; +import org.gcube.common.resources.gcore.Resources; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.Software; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.context.ContextElaborator; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.resources.discovery.icclient.ICFactory; +import org.gcube.testutility.ContextTest; +import org.junit.Assert; +import org.junit.Test; + +public class StatisticsAggregator extends ContextElaborator { + + private File directory; + private File statisticsFile; + + private Map contextCount; + + // > + private Map> globalMap; + + // > + private Map>> voContextMap; + private ScopeBean currentScopeBean; + + public StatisticsAggregator() { + contextCount = new HashMap<>(); + globalMap = new HashMap<>(); + voContextMap = new HashMap<>(); + directory = new File("/home/lucafrosini/Desktop/GlobalStatistics"); + if(!directory.exists()) { + directory.mkdirs(); + } + } + + private void printLineToFile(String line, File file) throws IOException { + synchronized(file) { + try(FileWriter fw = new FileWriter(file, true); + BufferedWriter bw = new BufferedWriter(fw); + PrintWriter out = new PrintWriter(bw)) { + out.println(line); + out.flush(); + } catch(IOException e) { + throw e; + } + } + } + + protected String getUnmarshalledResource(Resource gr) { + StringWriter stringWriter = new StringWriter(); + Resources.marshal(gr, stringWriter); + return stringWriter.toString(); + } + + private int getResourceSize(Resource r) throws UnsupportedEncodingException { + String unmarshalledR = getUnmarshalledResource(r); + final byte[] grUTF8Bytes = unmarshalledR.getBytes("UTF-8"); + return grUTF8Bytes.length; + } + + public void analizeInstances(Class clz) throws Exception { + Map resourcesGlobalMap = globalMap.get(clz.getSimpleName()); + Map resourcesPerContextMap = null; + Type type = currentScopeBean.type(); + + if(type != Type.INFRASTRUCTURE) { + resourcesPerContextMap = getVOContextMap().get(clz.getSimpleName()); + } + + DiscoveryClient client = ICFactory.clientFor(clz); + SimpleQuery query = ICFactory.queryFor(clz); + List instances = client.submit(query); + for(R r : instances) { + String id = r.id(); + int resourceSize = getResourceSize(r); + if(type != Type.INFRASTRUCTURE) { + if(!resourcesPerContextMap.containsKey(id)) { + resourcesPerContextMap.put(id, resourceSize); + } + } + + if(!resourcesGlobalMap.containsKey(id)) { + resourcesGlobalMap.put(id, resourceSize); + } + } + } + + public void collectStatistics() throws Exception { + List> classes = new ArrayList<>(); + classes.add(GCoreEndpoint.class); + classes.add(HostingNode.class); + classes.add(ServiceEndpoint.class); + classes.add(GenericResource.class); + classes.add(Software.class); + + for(Class clz : classes) { + Map resourcesGlobalMap = new HashMap<>(); + globalMap.put(clz.getSimpleName(), resourcesGlobalMap); + + if(currentScopeBean.type() != Type.INFRASTRUCTURE) { + Map> contextMap = getVOContextMap(); + Map resourcesPerContextMap = new HashMap<>(); + contextMap.put(clz.getSimpleName(), resourcesPerContextMap); + } + + analizeInstances(clz); + } + + } + + private Map> getVOContextMap() { + Map> resourcesPerVOMap = null; + Type type = currentScopeBean.type(); + String contextFullName = null; + switch(type) { + case VRE: + contextFullName = currentScopeBean.enclosingScope().toString(); + break; + case VO: + contextFullName = currentScopeBean.toString(); + break; + + default: + break; + } + + if(contextFullName != null) { + resourcesPerVOMap = voContextMap.get(contextFullName); + if(resourcesPerVOMap == null) { + resourcesPerVOMap = new HashMap<>(); + voContextMap.put(contextFullName, resourcesPerVOMap); + } + } + + return resourcesPerVOMap; + } + + public String generateUserToken(String context) throws Exception { + AuthorizationEntry authorizationEntry = Constants.authorizationService() + .get(SecurityTokenProvider.instance.get()); + UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo(); + String userToken = authorizationService().generateUserToken(userInfo, context); + logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken); + return userToken; + } + + private void generateStatistics() throws Exception { + statisticsFile = new File(directory, "all.csv"); + if(statisticsFile.exists()) { + statisticsFile.delete(); + } + statisticsFile.createNewFile(); + addContextCount(); + addGlobalStatistics(); + addVOBasedStatistics(); + } + + private void addContextCount() throws Exception { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("Context Type"); + stringBuffer.append(","); + stringBuffer.append("Count"); + printLineToFile(stringBuffer.toString(), statisticsFile); + + for(Type type : contextCount.keySet()) { + stringBuffer = new StringBuffer(); + stringBuffer.append(type.name()); + stringBuffer.append(","); + stringBuffer.append(contextCount.get(type)); + printLineToFile(stringBuffer.toString(), statisticsFile); + } + + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + } + + private void addHeader() throws IOException { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("Resource Type"); + stringBuffer.append(","); + stringBuffer.append("Count"); + stringBuffer.append(","); + stringBuffer.append("Avg Size"); + stringBuffer.append(","); + stringBuffer.append("Max"); + stringBuffer.append(","); + stringBuffer.append("Min"); + stringBuffer.append(","); + stringBuffer.append("Variance"); + printLineToFile(stringBuffer.toString(), statisticsFile); + } + + private void addResourceStatistic(String resourceType, SummaryStatistics summaryStatistics) throws IOException { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append(resourceType); + stringBuffer.append(","); + stringBuffer.append(summaryStatistics.getN()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getMean()).intValue()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getMax()).intValue()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getVariance()).intValue()); + printLineToFile(stringBuffer.toString(), statisticsFile); + } + + private void getStatistcsFromMap(Map> resourceMap) throws IOException { + for(String resourceType : resourceMap.keySet()) { + Map sizeMap = resourceMap.get(resourceType); + SummaryStatistics summaryStatistics = new SummaryStatistics(); + for(Integer integer : sizeMap.values()) { + summaryStatistics.addValue(integer); + } + Assert.assertTrue(resourceMap.size()==Double.valueOf(summaryStatistics.getN()).intValue()); + addResourceStatistic(resourceType, summaryStatistics); + } + } + + private void addGlobalStatistics() throws IOException { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("Whole Infrastructure Statistics"); + printLineToFile(stringBuffer.toString(), statisticsFile); + addHeader(); + getStatistcsFromMap(globalMap); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + } + + private void addVOBasedStatistics() throws Exception { + for(String vo : voContextMap.keySet()) { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append(vo); + stringBuffer.append(" Statistics"); + printLineToFile(stringBuffer.toString(), statisticsFile); + addHeader(); + getStatistcsFromMap(voContextMap.get(vo)); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + } + } + + @Override + public void elaborateContext(ScopeBean scopeBean) throws Exception { + this.currentScopeBean = scopeBean; + Type type = scopeBean.type(); + + Integer integer = contextCount.get(type); + if(integer == null) { + integer = 0; + } + contextCount.put(type, integer + 1); + + String contextFullName = scopeBean.toString(); + String token = generateUserToken(contextFullName); + ContextTest.setContext(token); + if(type == Type.VO) { + Map> contextMap = new HashMap<>(); + voContextMap.put(contextFullName, contextMap); + } + collectStatistics(); + } + + @Test + public void test() throws Exception { + ContextTest.setContextByName("/d4science.research-infrastructures.eu"); + all(); + generateStatistics(); + } + +}