301 lines
10 KiB
Java
301 lines
10 KiB
Java
package org.gcube.phd;
|
|
|
|
import java.io.BufferedWriter;
|
|
import java.io.File;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.PrintWriter;
|
|
import java.text.SimpleDateFormat;
|
|
import java.util.Calendar;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
import java.util.SortedMap;
|
|
import java.util.TreeMap;
|
|
|
|
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
|
|
import org.gcube.accounting.analytics.Info;
|
|
import org.gcube.accounting.analytics.TemporalConstraint;
|
|
import org.gcube.common.resources.gcore.Resource;
|
|
import org.gcube.common.scope.impl.ScopeBean;
|
|
import org.gcube.common.scope.impl.ScopeBean.Type;
|
|
import org.gcube.testutility.ContextTest;
|
|
import org.junit.Test;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
public class StatisticsGenerator {
|
|
|
|
protected Logger logger = LoggerFactory.getLogger(this.getClass());
|
|
|
|
private File directory;
|
|
private File statisticsFile;
|
|
|
|
public StatisticsGenerator() {
|
|
directory = new File("/home/lucafrosini/Dropbox/GlobalStatistics");
|
|
if(!directory.exists()) {
|
|
directory.mkdirs();
|
|
}
|
|
}
|
|
|
|
private void printLineToFile(String line, File file) throws IOException {
|
|
synchronized(file) {
|
|
try(FileWriter fw = new FileWriter(file, true);
|
|
BufferedWriter bw = new BufferedWriter(fw);
|
|
PrintWriter out = new PrintWriter(bw)) {
|
|
out.println(line);
|
|
out.flush();
|
|
} catch(IOException e) {
|
|
throw e;
|
|
}
|
|
}
|
|
}
|
|
|
|
private void addContextCount(Map<Type,Integer> contextCount) throws Exception {
|
|
StringBuffer stringBuffer = new StringBuffer();
|
|
stringBuffer.append("Context Type");
|
|
stringBuffer.append(",");
|
|
stringBuffer.append("Count");
|
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
|
|
|
for(Type type : Type.values()) {
|
|
stringBuffer = new StringBuffer();
|
|
stringBuffer.append(type.name());
|
|
stringBuffer.append(",");
|
|
stringBuffer.append(contextCount.get(type));
|
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
|
}
|
|
|
|
printLineToFile("", statisticsFile);
|
|
printLineToFile("", statisticsFile);
|
|
printLineToFile("", statisticsFile);
|
|
printLineToFile("", statisticsFile);
|
|
}
|
|
|
|
private void addHeader() throws IOException {
|
|
StringBuffer stringBuffer = new StringBuffer();
|
|
stringBuffer.append("Resource Type");
|
|
stringBuffer.append(",");
|
|
stringBuffer.append("Count");
|
|
stringBuffer.append(",");
|
|
stringBuffer.append("Avg Size");
|
|
stringBuffer.append(",");
|
|
stringBuffer.append("Max");
|
|
stringBuffer.append(",");
|
|
stringBuffer.append("Min");
|
|
stringBuffer.append(",");
|
|
stringBuffer.append("Standard Deviation");
|
|
stringBuffer.append(",");
|
|
stringBuffer.append("Geometric Mean");
|
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
|
}
|
|
|
|
private void addResourceStatistic(Class<? extends Resource> clz, SummaryStatistics summaryStatistics)
|
|
throws IOException {
|
|
StringBuffer stringBuffer = new StringBuffer();
|
|
stringBuffer.append(clz.getSimpleName());
|
|
stringBuffer.append(",");
|
|
stringBuffer.append(summaryStatistics.getN());
|
|
stringBuffer.append(",");
|
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMean()).intValue());
|
|
stringBuffer.append(",");
|
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMax()).intValue());
|
|
stringBuffer.append(",");
|
|
stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue());
|
|
stringBuffer.append(",");
|
|
stringBuffer.append(Double.valueOf(summaryStatistics.getStandardDeviation()).intValue());
|
|
stringBuffer.append(",");
|
|
stringBuffer.append(Double.valueOf(summaryStatistics.getGeometricMean()).intValue());
|
|
|
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
|
}
|
|
|
|
private void addStatistics(Map<Class<? extends Resource>,SummaryStatistics> statistics) throws Exception {
|
|
for(Class<? extends Resource> clz : statistics.keySet()) {
|
|
addResourceStatistic(clz, statistics.get(clz));
|
|
}
|
|
}
|
|
|
|
private void contextStatisticReport(ScopeBean scopeBean, Map<Class<? extends Resource>,SummaryStatistics> statistics) throws Exception {
|
|
StringBuffer stringBuffer = new StringBuffer();
|
|
stringBuffer.append(scopeBean.toString());
|
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
|
addHeader();
|
|
|
|
addStatistics(statistics);
|
|
}
|
|
|
|
@SuppressWarnings("unused")
|
|
private void contextAccountingReport(ScopeBean scopeBean, SortedMap<Calendar,Info> accountingData) throws Exception {
|
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
simpleDateFormat.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
|
|
StringBuffer calendarBuffer = new StringBuffer();
|
|
StringBuffer dataBuffer = new StringBuffer();
|
|
boolean first = true;
|
|
for(Calendar calendar : accountingData.keySet()) {
|
|
if(first) {
|
|
first = false;
|
|
}else {
|
|
calendarBuffer.append(",");
|
|
dataBuffer.append(",");
|
|
}
|
|
calendarBuffer.append(simpleDateFormat.format(calendar.getTime()));
|
|
Info info = accountingData.get(calendar);
|
|
int operationCount = (int) info.getValue().get("operationCount");
|
|
dataBuffer.append(operationCount);
|
|
}
|
|
printLineToFile(calendarBuffer.toString(), statisticsFile);
|
|
printLineToFile(dataBuffer.toString(), statisticsFile);
|
|
}
|
|
|
|
private void generateAggregatedStatistics(
|
|
SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources)
|
|
throws Exception {
|
|
|
|
printLineToFile("", statisticsFile);
|
|
printLineToFile("", statisticsFile);
|
|
printLineToFile("", statisticsFile);
|
|
printLineToFile("######################################", statisticsFile);
|
|
printLineToFile("Aggregated Statistics", statisticsFile);
|
|
|
|
for(ScopeBean scopeBean : aggregatedResources.keySet()) {
|
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap = aggregatedResources
|
|
.get(scopeBean);
|
|
Map<Class<? extends Resource>,SummaryStatistics> statistics = ContextStatistic
|
|
.generateStatistics(resourceMap);
|
|
|
|
contextStatisticReport(scopeBean, statistics);
|
|
|
|
}
|
|
}
|
|
|
|
|
|
private void generateContextStatisticReport(ScopeBean scopeBean, ContextStatistic contextStatistic) throws Exception {
|
|
contextStatisticReport(scopeBean, contextStatistic.getStatistics());
|
|
/*
|
|
try {
|
|
contextAccountingReport(scopeBean, contextStatistic.getAccountingData());
|
|
}catch (Exception e) {
|
|
logger.error("Unable to Generate AccountignData resport for contes {}", scopeBean.toString(), e);
|
|
printLineToFile("PARTIAL OR INVALID ACCOUNTING DATA", statisticsFile);
|
|
}
|
|
*/
|
|
printLineToFile("", statisticsFile);
|
|
printLineToFile("", statisticsFile);
|
|
}
|
|
|
|
private SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> getAggregatedResources(
|
|
SortedMap<ScopeBean,ContextStatistic> contextStatistics) throws Exception {
|
|
|
|
SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources = new TreeMap<>(
|
|
new ScopeBeanComparator());
|
|
|
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> infrastructureResourcesByClass = null;
|
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> voResourcesByClass = null;
|
|
|
|
for(ScopeBean scopeBean : contextStatistics.keySet()) {
|
|
|
|
ContextStatistic contextStatistic = contextStatistics.get(scopeBean);
|
|
generateContextStatisticReport(scopeBean, contextStatistic);
|
|
|
|
|
|
logger.debug("Generating AggregatedStatistics for {}", scopeBean.toString());
|
|
|
|
Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap = contextStatistic
|
|
.getAllResources();
|
|
|
|
Type type = scopeBean.type();
|
|
switch(type) {
|
|
case INFRASTRUCTURE:
|
|
infrastructureResourcesByClass = new HashMap<>();
|
|
aggregatedResources.put(scopeBean, infrastructureResourcesByClass);
|
|
break;
|
|
|
|
case VO:
|
|
voResourcesByClass = new HashMap<>();
|
|
aggregatedResources.put(scopeBean, voResourcesByClass);
|
|
break;
|
|
|
|
case VRE:
|
|
break;
|
|
|
|
default:
|
|
break;
|
|
}
|
|
|
|
File dir = new File(directory, scopeBean.toString());
|
|
if(!dir.exists()) {
|
|
dir.mkdirs();
|
|
}
|
|
|
|
for(Class<? extends Resource> clz : resourceMap.keySet()) {
|
|
Set<ResourceInfo<? extends Resource>> resources = resourceMap.get(clz);
|
|
/*
|
|
File resourceDir = new File(dir, "__Resources");
|
|
resourceDir = new File(resourceDir, clz.getSimpleName());
|
|
if(!resourceDir.exists()) {
|
|
resourceDir.mkdirs();
|
|
}
|
|
for(ResourceInfo<? extends Resource> resourceInfo : resources) {
|
|
File f = new File(resourceDir, resourceInfo.getResourceID());
|
|
if(f.exists()) {
|
|
f.delete();
|
|
}
|
|
f.createNewFile();
|
|
printLineToFile(resourceInfo.getUnmarshalledResource(),f);
|
|
}
|
|
*/
|
|
|
|
|
|
Set<ResourceInfo<? extends Resource>> infrastructureResources = infrastructureResourcesByClass.get(clz);
|
|
if(infrastructureResources == null) {
|
|
infrastructureResources = new HashSet<>();
|
|
infrastructureResourcesByClass.put(clz, infrastructureResources);
|
|
}
|
|
infrastructureResources.addAll(resources);
|
|
|
|
if(type != Type.INFRASTRUCTURE) {
|
|
Set<ResourceInfo<? extends Resource>> voResources = voResourcesByClass.get(clz);
|
|
if(voResources == null) {
|
|
voResources = new HashSet<>();
|
|
voResourcesByClass.put(clz, voResources);
|
|
}
|
|
voResources.addAll(resources);
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return aggregatedResources;
|
|
|
|
}
|
|
|
|
|
|
|
|
private void generateStatistics() throws Exception {
|
|
statisticsFile = new File(directory, "all-corrected.csv");
|
|
if(statisticsFile.exists()) {
|
|
statisticsFile.delete();
|
|
}
|
|
statisticsFile.createNewFile();
|
|
|
|
StatisticsCollector statisticsCollector = new StatisticsCollector();
|
|
Map<Type,Integer> contextCount = statisticsCollector.getContextCount();
|
|
addContextCount(contextCount);
|
|
|
|
SortedMap<ScopeBean,ContextStatistic> contextStatistics = statisticsCollector.getStatistics();
|
|
SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources = getAggregatedResources(
|
|
contextStatistics);
|
|
generateAggregatedStatistics(aggregatedResources);
|
|
}
|
|
|
|
@Test
|
|
public void test() throws Exception {
|
|
ContextTest.setContextByName("/d4science.research-infrastructures.eu");
|
|
generateStatistics();
|
|
}
|
|
|
|
}
|