git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/private/luca.frosini/infrastructure-tests@178411 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
5966c5c151
commit
8ec41a6d80
|
@ -32,13 +32,13 @@ import org.gcube.resources.discovery.icclient.ICFactory;
|
||||||
import org.gcube.testutility.ContextTest;
|
import org.gcube.testutility.ContextTest;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
public class GlobalStatistics extends ContextElaborator {
|
public class OLDGlobalStatistics extends ContextElaborator {
|
||||||
|
|
||||||
private File directory;
|
private File directory;
|
||||||
|
|
||||||
private Set<String> visited;
|
private Set<String> visited;
|
||||||
|
|
||||||
public GlobalStatistics() {
|
public OLDGlobalStatistics() {
|
||||||
visited = new HashSet<String>();
|
visited = new HashSet<String>();
|
||||||
directory = new File("/home/lucafrosini/Desktop/Statistiche/Global");
|
directory = new File("/home/lucafrosini/Desktop/Statistiche/Global");
|
||||||
if(!directory.exists()) {
|
if(!directory.exists()) {
|
||||||
|
@ -71,7 +71,7 @@ public class GlobalStatistics extends ContextElaborator {
|
||||||
if(!statisticsFile.exists()) {
|
if(!statisticsFile.exists()) {
|
||||||
statisticsFile.createNewFile();
|
statisticsFile.createNewFile();
|
||||||
stringBuffer.append(r.getClass().getSimpleName());
|
stringBuffer.append(r.getClass().getSimpleName());
|
||||||
stringBuffer.append("(byte size)");
|
stringBuffer.append(" (byte)");
|
||||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
stringBuffer = new StringBuffer();
|
stringBuffer = new StringBuffer();
|
||||||
}
|
}
|
|
@ -0,0 +1,304 @@
|
||||||
|
package org.gcube.phd;
|
||||||
|
|
||||||
|
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||||
|
|
||||||
|
import java.io.BufferedWriter;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileWriter;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.io.StringWriter;
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
|
||||||
|
import org.gcube.common.authorization.client.Constants;
|
||||||
|
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||||
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.common.authorization.library.provider.UserInfo;
|
||||||
|
import org.gcube.common.resources.gcore.GCoreEndpoint;
|
||||||
|
import org.gcube.common.resources.gcore.GenericResource;
|
||||||
|
import org.gcube.common.resources.gcore.HostingNode;
|
||||||
|
import org.gcube.common.resources.gcore.Resource;
|
||||||
|
import org.gcube.common.resources.gcore.Resources;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||||
|
import org.gcube.common.resources.gcore.Software;
|
||||||
|
import org.gcube.common.scope.impl.ScopeBean;
|
||||||
|
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||||
|
import org.gcube.context.ContextElaborator;
|
||||||
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||||
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||||
|
import org.gcube.resources.discovery.icclient.ICFactory;
|
||||||
|
import org.gcube.testutility.ContextTest;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class StatisticsAggregator extends ContextElaborator {
|
||||||
|
|
||||||
|
private File directory;
|
||||||
|
private File statisticsFile;
|
||||||
|
|
||||||
|
private Map<Type,Integer> contextCount;
|
||||||
|
|
||||||
|
// <resourcetype,<id,size>>
|
||||||
|
private Map<String,Map<String,Integer>> globalMap;
|
||||||
|
|
||||||
|
// <VOcontext,<resourcetype,<id,size>>
|
||||||
|
private Map<String,Map<String,Map<String,Integer>>> voContextMap;
|
||||||
|
private ScopeBean currentScopeBean;
|
||||||
|
|
||||||
|
public StatisticsAggregator() {
|
||||||
|
contextCount = new HashMap<>();
|
||||||
|
globalMap = new HashMap<>();
|
||||||
|
voContextMap = new HashMap<>();
|
||||||
|
directory = new File("/home/lucafrosini/Desktop/GlobalStatistics");
|
||||||
|
if(!directory.exists()) {
|
||||||
|
directory.mkdirs();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void printLineToFile(String line, File file) throws IOException {
|
||||||
|
synchronized(file) {
|
||||||
|
try(FileWriter fw = new FileWriter(file, true);
|
||||||
|
BufferedWriter bw = new BufferedWriter(fw);
|
||||||
|
PrintWriter out = new PrintWriter(bw)) {
|
||||||
|
out.println(line);
|
||||||
|
out.flush();
|
||||||
|
} catch(IOException e) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String getUnmarshalledResource(Resource gr) {
|
||||||
|
StringWriter stringWriter = new StringWriter();
|
||||||
|
Resources.marshal(gr, stringWriter);
|
||||||
|
return stringWriter.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private int getResourceSize(Resource r) throws UnsupportedEncodingException {
|
||||||
|
String unmarshalledR = getUnmarshalledResource(r);
|
||||||
|
final byte[] grUTF8Bytes = unmarshalledR.getBytes("UTF-8");
|
||||||
|
return grUTF8Bytes.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
public <R extends Resource> void analizeInstances(Class<R> clz) throws Exception {
|
||||||
|
Map<String,Integer> resourcesGlobalMap = globalMap.get(clz.getSimpleName());
|
||||||
|
Map<String,Integer> resourcesPerContextMap = null;
|
||||||
|
Type type = currentScopeBean.type();
|
||||||
|
|
||||||
|
if(type != Type.INFRASTRUCTURE) {
|
||||||
|
resourcesPerContextMap = getVOContextMap().get(clz.getSimpleName());
|
||||||
|
}
|
||||||
|
|
||||||
|
DiscoveryClient<R> client = ICFactory.clientFor(clz);
|
||||||
|
SimpleQuery query = ICFactory.queryFor(clz);
|
||||||
|
List<R> instances = client.submit(query);
|
||||||
|
for(R r : instances) {
|
||||||
|
String id = r.id();
|
||||||
|
int resourceSize = getResourceSize(r);
|
||||||
|
if(type != Type.INFRASTRUCTURE) {
|
||||||
|
if(!resourcesPerContextMap.containsKey(id)) {
|
||||||
|
resourcesPerContextMap.put(id, resourceSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!resourcesGlobalMap.containsKey(id)) {
|
||||||
|
resourcesGlobalMap.put(id, resourceSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void collectStatistics() throws Exception {
|
||||||
|
List<Class<? extends Resource>> classes = new ArrayList<>();
|
||||||
|
classes.add(GCoreEndpoint.class);
|
||||||
|
classes.add(HostingNode.class);
|
||||||
|
classes.add(ServiceEndpoint.class);
|
||||||
|
classes.add(GenericResource.class);
|
||||||
|
classes.add(Software.class);
|
||||||
|
|
||||||
|
for(Class<? extends Resource> clz : classes) {
|
||||||
|
Map<String,Integer> resourcesGlobalMap = new HashMap<>();
|
||||||
|
globalMap.put(clz.getSimpleName(), resourcesGlobalMap);
|
||||||
|
|
||||||
|
if(currentScopeBean.type() != Type.INFRASTRUCTURE) {
|
||||||
|
Map<String,Map<String,Integer>> contextMap = getVOContextMap();
|
||||||
|
Map<String,Integer> resourcesPerContextMap = new HashMap<>();
|
||||||
|
contextMap.put(clz.getSimpleName(), resourcesPerContextMap);
|
||||||
|
}
|
||||||
|
|
||||||
|
analizeInstances(clz);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String,Map<String,Integer>> getVOContextMap() {
|
||||||
|
Map<String,Map<String,Integer>> resourcesPerVOMap = null;
|
||||||
|
Type type = currentScopeBean.type();
|
||||||
|
String contextFullName = null;
|
||||||
|
switch(type) {
|
||||||
|
case VRE:
|
||||||
|
contextFullName = currentScopeBean.enclosingScope().toString();
|
||||||
|
break;
|
||||||
|
case VO:
|
||||||
|
contextFullName = currentScopeBean.toString();
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(contextFullName != null) {
|
||||||
|
resourcesPerVOMap = voContextMap.get(contextFullName);
|
||||||
|
if(resourcesPerVOMap == null) {
|
||||||
|
resourcesPerVOMap = new HashMap<>();
|
||||||
|
voContextMap.put(contextFullName, resourcesPerVOMap);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return resourcesPerVOMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String generateUserToken(String context) throws Exception {
|
||||||
|
AuthorizationEntry authorizationEntry = Constants.authorizationService()
|
||||||
|
.get(SecurityTokenProvider.instance.get());
|
||||||
|
UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo();
|
||||||
|
String userToken = authorizationService().generateUserToken(userInfo, context);
|
||||||
|
logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken);
|
||||||
|
return userToken;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void generateStatistics() throws Exception {
|
||||||
|
statisticsFile = new File(directory, "all.csv");
|
||||||
|
if(statisticsFile.exists()) {
|
||||||
|
statisticsFile.delete();
|
||||||
|
}
|
||||||
|
statisticsFile.createNewFile();
|
||||||
|
addContextCount();
|
||||||
|
addGlobalStatistics();
|
||||||
|
addVOBasedStatistics();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addContextCount() throws Exception {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append("Context Type");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Count");
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
|
||||||
|
for(Type type : contextCount.keySet()) {
|
||||||
|
stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append(type.name());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(contextCount.get(type));
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addHeader() throws IOException {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append("Resource Type");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Count");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Avg Size");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Max");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Min");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append("Variance");
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addResourceStatistic(String resourceType, SummaryStatistics summaryStatistics) throws IOException {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append(resourceType);
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(summaryStatistics.getN());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMean()).intValue());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMax()).intValue());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue());
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(Double.valueOf(summaryStatistics.getVariance()).intValue());
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void getStatistcsFromMap(Map<String,Map<String,Integer>> resourceMap) throws IOException {
|
||||||
|
for(String resourceType : resourceMap.keySet()) {
|
||||||
|
Map<String,Integer> sizeMap = resourceMap.get(resourceType);
|
||||||
|
SummaryStatistics summaryStatistics = new SummaryStatistics();
|
||||||
|
for(Integer integer : sizeMap.values()) {
|
||||||
|
summaryStatistics.addValue(integer);
|
||||||
|
}
|
||||||
|
Assert.assertTrue(resourceMap.size()==Double.valueOf(summaryStatistics.getN()).intValue());
|
||||||
|
addResourceStatistic(resourceType, summaryStatistics);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addGlobalStatistics() throws IOException {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append("Whole Infrastructure Statistics");
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
addHeader();
|
||||||
|
getStatistcsFromMap(globalMap);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addVOBasedStatistics() throws Exception {
|
||||||
|
for(String vo : voContextMap.keySet()) {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append(vo);
|
||||||
|
stringBuffer.append(" Statistics");
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
addHeader();
|
||||||
|
getStatistcsFromMap(voContextMap.get(vo));
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
printLineToFile("", statisticsFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void elaborateContext(ScopeBean scopeBean) throws Exception {
|
||||||
|
this.currentScopeBean = scopeBean;
|
||||||
|
Type type = scopeBean.type();
|
||||||
|
|
||||||
|
Integer integer = contextCount.get(type);
|
||||||
|
if(integer == null) {
|
||||||
|
integer = 0;
|
||||||
|
}
|
||||||
|
contextCount.put(type, integer + 1);
|
||||||
|
|
||||||
|
String contextFullName = scopeBean.toString();
|
||||||
|
String token = generateUserToken(contextFullName);
|
||||||
|
ContextTest.setContext(token);
|
||||||
|
if(type == Type.VO) {
|
||||||
|
Map<String,Map<String,Integer>> contextMap = new HashMap<>();
|
||||||
|
voContextMap.put(contextFullName, contextMap);
|
||||||
|
}
|
||||||
|
collectStatistics();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void test() throws Exception {
|
||||||
|
ContextTest.setContextByName("/d4science.research-infrastructures.eu");
|
||||||
|
all();
|
||||||
|
generateStatistics();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue