git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/private/luca.frosini/infrastructure-tests@178411 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
5966c5c151
commit
8ec41a6d80
|
|
@ -32,13 +32,13 @@ import org.gcube.resources.discovery.icclient.ICFactory;
|
|||
import org.gcube.testutility.ContextTest;
|
||||
import org.junit.Test;
|
||||
|
||||
public class GlobalStatistics extends ContextElaborator {
|
||||
public class OLDGlobalStatistics extends ContextElaborator {
|
||||
|
||||
private File directory;
|
||||
|
||||
private Set<String> visited;
|
||||
|
||||
public GlobalStatistics() {
|
||||
public OLDGlobalStatistics() {
|
||||
visited = new HashSet<String>();
|
||||
directory = new File("/home/lucafrosini/Desktop/Statistiche/Global");
|
||||
if(!directory.exists()) {
|
||||
|
|
@ -71,7 +71,7 @@ public class GlobalStatistics extends ContextElaborator {
|
|||
if(!statisticsFile.exists()) {
|
||||
statisticsFile.createNewFile();
|
||||
stringBuffer.append(r.getClass().getSimpleName());
|
||||
stringBuffer.append("(byte size)");
|
||||
stringBuffer.append(" (byte)");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
stringBuffer = new StringBuffer();
|
||||
}
|
||||
|
|
@ -0,0 +1,304 @@
|
|||
package org.gcube.phd;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
|
||||
import org.gcube.common.authorization.client.Constants;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.authorization.library.provider.UserInfo;
|
||||
import org.gcube.common.resources.gcore.GCoreEndpoint;
|
||||
import org.gcube.common.resources.gcore.GenericResource;
|
||||
import org.gcube.common.resources.gcore.HostingNode;
|
||||
import org.gcube.common.resources.gcore.Resource;
|
||||
import org.gcube.common.resources.gcore.Resources;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||
import org.gcube.common.resources.gcore.Software;
|
||||
import org.gcube.common.scope.impl.ScopeBean;
|
||||
import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||
import org.gcube.context.ContextElaborator;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
import org.gcube.resources.discovery.icclient.ICFactory;
|
||||
import org.gcube.testutility.ContextTest;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class StatisticsAggregator extends ContextElaborator {
|
||||
|
||||
private File directory;
|
||||
private File statisticsFile;
|
||||
|
||||
private Map<Type,Integer> contextCount;
|
||||
|
||||
// <resourcetype,<id,size>>
|
||||
private Map<String,Map<String,Integer>> globalMap;
|
||||
|
||||
// <VOcontext,<resourcetype,<id,size>>
|
||||
private Map<String,Map<String,Map<String,Integer>>> voContextMap;
|
||||
private ScopeBean currentScopeBean;
|
||||
|
||||
public StatisticsAggregator() {
|
||||
contextCount = new HashMap<>();
|
||||
globalMap = new HashMap<>();
|
||||
voContextMap = new HashMap<>();
|
||||
directory = new File("/home/lucafrosini/Desktop/GlobalStatistics");
|
||||
if(!directory.exists()) {
|
||||
directory.mkdirs();
|
||||
}
|
||||
}
|
||||
|
||||
private void printLineToFile(String line, File file) throws IOException {
|
||||
synchronized(file) {
|
||||
try(FileWriter fw = new FileWriter(file, true);
|
||||
BufferedWriter bw = new BufferedWriter(fw);
|
||||
PrintWriter out = new PrintWriter(bw)) {
|
||||
out.println(line);
|
||||
out.flush();
|
||||
} catch(IOException e) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected String getUnmarshalledResource(Resource gr) {
|
||||
StringWriter stringWriter = new StringWriter();
|
||||
Resources.marshal(gr, stringWriter);
|
||||
return stringWriter.toString();
|
||||
}
|
||||
|
||||
private int getResourceSize(Resource r) throws UnsupportedEncodingException {
|
||||
String unmarshalledR = getUnmarshalledResource(r);
|
||||
final byte[] grUTF8Bytes = unmarshalledR.getBytes("UTF-8");
|
||||
return grUTF8Bytes.length;
|
||||
}
|
||||
|
||||
public <R extends Resource> void analizeInstances(Class<R> clz) throws Exception {
|
||||
Map<String,Integer> resourcesGlobalMap = globalMap.get(clz.getSimpleName());
|
||||
Map<String,Integer> resourcesPerContextMap = null;
|
||||
Type type = currentScopeBean.type();
|
||||
|
||||
if(type != Type.INFRASTRUCTURE) {
|
||||
resourcesPerContextMap = getVOContextMap().get(clz.getSimpleName());
|
||||
}
|
||||
|
||||
DiscoveryClient<R> client = ICFactory.clientFor(clz);
|
||||
SimpleQuery query = ICFactory.queryFor(clz);
|
||||
List<R> instances = client.submit(query);
|
||||
for(R r : instances) {
|
||||
String id = r.id();
|
||||
int resourceSize = getResourceSize(r);
|
||||
if(type != Type.INFRASTRUCTURE) {
|
||||
if(!resourcesPerContextMap.containsKey(id)) {
|
||||
resourcesPerContextMap.put(id, resourceSize);
|
||||
}
|
||||
}
|
||||
|
||||
if(!resourcesGlobalMap.containsKey(id)) {
|
||||
resourcesGlobalMap.put(id, resourceSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void collectStatistics() throws Exception {
|
||||
List<Class<? extends Resource>> classes = new ArrayList<>();
|
||||
classes.add(GCoreEndpoint.class);
|
||||
classes.add(HostingNode.class);
|
||||
classes.add(ServiceEndpoint.class);
|
||||
classes.add(GenericResource.class);
|
||||
classes.add(Software.class);
|
||||
|
||||
for(Class<? extends Resource> clz : classes) {
|
||||
Map<String,Integer> resourcesGlobalMap = new HashMap<>();
|
||||
globalMap.put(clz.getSimpleName(), resourcesGlobalMap);
|
||||
|
||||
if(currentScopeBean.type() != Type.INFRASTRUCTURE) {
|
||||
Map<String,Map<String,Integer>> contextMap = getVOContextMap();
|
||||
Map<String,Integer> resourcesPerContextMap = new HashMap<>();
|
||||
contextMap.put(clz.getSimpleName(), resourcesPerContextMap);
|
||||
}
|
||||
|
||||
analizeInstances(clz);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private Map<String,Map<String,Integer>> getVOContextMap() {
|
||||
Map<String,Map<String,Integer>> resourcesPerVOMap = null;
|
||||
Type type = currentScopeBean.type();
|
||||
String contextFullName = null;
|
||||
switch(type) {
|
||||
case VRE:
|
||||
contextFullName = currentScopeBean.enclosingScope().toString();
|
||||
break;
|
||||
case VO:
|
||||
contextFullName = currentScopeBean.toString();
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if(contextFullName != null) {
|
||||
resourcesPerVOMap = voContextMap.get(contextFullName);
|
||||
if(resourcesPerVOMap == null) {
|
||||
resourcesPerVOMap = new HashMap<>();
|
||||
voContextMap.put(contextFullName, resourcesPerVOMap);
|
||||
}
|
||||
}
|
||||
|
||||
return resourcesPerVOMap;
|
||||
}
|
||||
|
||||
public String generateUserToken(String context) throws Exception {
|
||||
AuthorizationEntry authorizationEntry = Constants.authorizationService()
|
||||
.get(SecurityTokenProvider.instance.get());
|
||||
UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo();
|
||||
String userToken = authorizationService().generateUserToken(userInfo, context);
|
||||
logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken);
|
||||
return userToken;
|
||||
}
|
||||
|
||||
private void generateStatistics() throws Exception {
|
||||
statisticsFile = new File(directory, "all.csv");
|
||||
if(statisticsFile.exists()) {
|
||||
statisticsFile.delete();
|
||||
}
|
||||
statisticsFile.createNewFile();
|
||||
addContextCount();
|
||||
addGlobalStatistics();
|
||||
addVOBasedStatistics();
|
||||
}
|
||||
|
||||
private void addContextCount() throws Exception {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append("Context Type");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append("Count");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
|
||||
for(Type type : contextCount.keySet()) {
|
||||
stringBuffer = new StringBuffer();
|
||||
stringBuffer.append(type.name());
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(contextCount.get(type));
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
}
|
||||
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
}
|
||||
|
||||
private void addHeader() throws IOException {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append("Resource Type");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append("Count");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append("Avg Size");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append("Max");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append("Min");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append("Variance");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
}
|
||||
|
||||
private void addResourceStatistic(String resourceType, SummaryStatistics summaryStatistics) throws IOException {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append(resourceType);
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(summaryStatistics.getN());
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(Double.valueOf(summaryStatistics.getMean()).intValue());
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(Double.valueOf(summaryStatistics.getMax()).intValue());
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue());
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(Double.valueOf(summaryStatistics.getVariance()).intValue());
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
}
|
||||
|
||||
private void getStatistcsFromMap(Map<String,Map<String,Integer>> resourceMap) throws IOException {
|
||||
for(String resourceType : resourceMap.keySet()) {
|
||||
Map<String,Integer> sizeMap = resourceMap.get(resourceType);
|
||||
SummaryStatistics summaryStatistics = new SummaryStatistics();
|
||||
for(Integer integer : sizeMap.values()) {
|
||||
summaryStatistics.addValue(integer);
|
||||
}
|
||||
Assert.assertTrue(resourceMap.size()==Double.valueOf(summaryStatistics.getN()).intValue());
|
||||
addResourceStatistic(resourceType, summaryStatistics);
|
||||
}
|
||||
}
|
||||
|
||||
private void addGlobalStatistics() throws IOException {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append("Whole Infrastructure Statistics");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
addHeader();
|
||||
getStatistcsFromMap(globalMap);
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
}
|
||||
|
||||
private void addVOBasedStatistics() throws Exception {
|
||||
for(String vo : voContextMap.keySet()) {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append(vo);
|
||||
stringBuffer.append(" Statistics");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
addHeader();
|
||||
getStatistcsFromMap(voContextMap.get(vo));
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
printLineToFile("", statisticsFile);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void elaborateContext(ScopeBean scopeBean) throws Exception {
|
||||
this.currentScopeBean = scopeBean;
|
||||
Type type = scopeBean.type();
|
||||
|
||||
Integer integer = contextCount.get(type);
|
||||
if(integer == null) {
|
||||
integer = 0;
|
||||
}
|
||||
contextCount.put(type, integer + 1);
|
||||
|
||||
String contextFullName = scopeBean.toString();
|
||||
String token = generateUserToken(contextFullName);
|
||||
ContextTest.setContext(token);
|
||||
if(type == Type.VO) {
|
||||
Map<String,Map<String,Integer>> contextMap = new HashMap<>();
|
||||
voContextMap.put(contextFullName, contextMap);
|
||||
}
|
||||
collectStatistics();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
ContextTest.setContextByName("/d4science.research-infrastructures.eu");
|
||||
all();
|
||||
generateStatistics();
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue