package org.gcube.phd; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.common.resources.gcore.Resource; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.testutility.ContextTest; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class StatisticsGenerator { protected Logger logger = LoggerFactory.getLogger(this.getClass()); private File directory; private File statisticsFile; public StatisticsGenerator() { directory = new File("/home/lucafrosini/Dropbox/GlobalStatistics"); if(!directory.exists()) { directory.mkdirs(); } } private void printLineToFile(String line, File file) throws IOException { synchronized(file) { try(FileWriter fw = new FileWriter(file, true); BufferedWriter bw = new BufferedWriter(fw); PrintWriter out = new PrintWriter(bw)) { out.println(line); out.flush(); } catch(IOException e) { throw e; } } } private void addContextCount(Map contextCount) throws Exception { StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append("Context Type"); stringBuffer.append(","); stringBuffer.append("Count"); printLineToFile(stringBuffer.toString(), statisticsFile); for(Type type : Type.values()) { stringBuffer = new StringBuffer(); stringBuffer.append(type.name()); stringBuffer.append(","); stringBuffer.append(contextCount.get(type)); printLineToFile(stringBuffer.toString(), statisticsFile); } printLineToFile("", statisticsFile); printLineToFile("", statisticsFile); printLineToFile("", statisticsFile); printLineToFile("", statisticsFile); } private void addHeader() throws IOException { StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append("Resource Type"); stringBuffer.append(","); stringBuffer.append("Count"); stringBuffer.append(","); stringBuffer.append("Avg Size"); stringBuffer.append(","); stringBuffer.append("Max"); stringBuffer.append(","); stringBuffer.append("Min"); stringBuffer.append(","); stringBuffer.append("Standard Deviation"); stringBuffer.append(","); stringBuffer.append("Geometric Mean"); printLineToFile(stringBuffer.toString(), statisticsFile); } private void addResourceStatistic(Class clz, SummaryStatistics summaryStatistics) throws IOException { StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append(clz.getSimpleName()); stringBuffer.append(","); stringBuffer.append(summaryStatistics.getN()); stringBuffer.append(","); stringBuffer.append(Double.valueOf(summaryStatistics.getMean()).intValue()); stringBuffer.append(","); stringBuffer.append(Double.valueOf(summaryStatistics.getMax()).intValue()); stringBuffer.append(","); stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue()); stringBuffer.append(","); stringBuffer.append(Double.valueOf(summaryStatistics.getStandardDeviation()).intValue()); stringBuffer.append(","); stringBuffer.append(Double.valueOf(summaryStatistics.getGeometricMean()).intValue()); printLineToFile(stringBuffer.toString(), statisticsFile); } private void addStatistics(Map,SummaryStatistics> statistics) throws Exception { for(Class clz : statistics.keySet()) { addResourceStatistic(clz, statistics.get(clz)); } } private void contextStatisticReport(ScopeBean scopeBean, Map,SummaryStatistics> statistics) throws Exception { StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append(scopeBean.toString()); printLineToFile(stringBuffer.toString(), statisticsFile); addHeader(); addStatistics(statistics); } @SuppressWarnings("unused") private void contextAccountingReport(ScopeBean scopeBean, SortedMap accountingData) throws Exception { SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); simpleDateFormat.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); StringBuffer calendarBuffer = new StringBuffer(); StringBuffer dataBuffer = new StringBuffer(); boolean first = true; for(Calendar calendar : accountingData.keySet()) { if(first) { first = false; }else { calendarBuffer.append(","); dataBuffer.append(","); } calendarBuffer.append(simpleDateFormat.format(calendar.getTime())); Info info = accountingData.get(calendar); int operationCount = (int) info.getValue().get("operationCount"); dataBuffer.append(operationCount); } printLineToFile(calendarBuffer.toString(), statisticsFile); printLineToFile(dataBuffer.toString(), statisticsFile); } private void generateAggregatedStatistics( SortedMap,Set>>> aggregatedResources) throws Exception { printLineToFile("", statisticsFile); printLineToFile("", statisticsFile); printLineToFile("", statisticsFile); printLineToFile("######################################", statisticsFile); printLineToFile("Aggregated Statistics", statisticsFile); for(ScopeBean scopeBean : aggregatedResources.keySet()) { Map,Set>> resourceMap = aggregatedResources .get(scopeBean); Map,SummaryStatistics> statistics = ContextStatistic .generateStatistics(resourceMap); contextStatisticReport(scopeBean, statistics); } } private void generateContextStatisticReport(ScopeBean scopeBean, ContextStatistic contextStatistic) throws Exception { contextStatisticReport(scopeBean, contextStatistic.getStatistics()); /* try { contextAccountingReport(scopeBean, contextStatistic.getAccountingData()); }catch (Exception e) { logger.error("Unable to Generate AccountignData resport for contes {}", scopeBean.toString(), e); printLineToFile("PARTIAL OR INVALID ACCOUNTING DATA", statisticsFile); } */ printLineToFile("", statisticsFile); printLineToFile("", statisticsFile); } private SortedMap,Set>>> getAggregatedResources( SortedMap contextStatistics) throws Exception { SortedMap,Set>>> aggregatedResources = new TreeMap<>( new ScopeBeanComparator()); Map,Set>> infrastructureResourcesByClass = null; Map,Set>> voResourcesByClass = null; for(ScopeBean scopeBean : contextStatistics.keySet()) { ContextStatistic contextStatistic = contextStatistics.get(scopeBean); generateContextStatisticReport(scopeBean, contextStatistic); logger.debug("Generating AggregatedStatistics for {}", scopeBean.toString()); Map,Set>> resourceMap = contextStatistic .getAllResources(); Type type = scopeBean.type(); switch(type) { case INFRASTRUCTURE: infrastructureResourcesByClass = new HashMap<>(); aggregatedResources.put(scopeBean, infrastructureResourcesByClass); break; case VO: voResourcesByClass = new HashMap<>(); aggregatedResources.put(scopeBean, voResourcesByClass); break; case VRE: break; default: break; } File dir = new File(directory, scopeBean.toString()); if(!dir.exists()) { dir.mkdirs(); } for(Class clz : resourceMap.keySet()) { Set> resources = resourceMap.get(clz); /* File resourceDir = new File(dir, "__Resources"); resourceDir = new File(resourceDir, clz.getSimpleName()); if(!resourceDir.exists()) { resourceDir.mkdirs(); } for(ResourceInfo resourceInfo : resources) { File f = new File(resourceDir, resourceInfo.getResourceID()); if(f.exists()) { f.delete(); } f.createNewFile(); printLineToFile(resourceInfo.getUnmarshalledResource(),f); } */ Set> infrastructureResources = infrastructureResourcesByClass.get(clz); if(infrastructureResources == null) { infrastructureResources = new HashSet<>(); infrastructureResourcesByClass.put(clz, infrastructureResources); } infrastructureResources.addAll(resources); if(type != Type.INFRASTRUCTURE) { Set> voResources = voResourcesByClass.get(clz); if(voResources == null) { voResources = new HashSet<>(); voResourcesByClass.put(clz, voResources); } voResources.addAll(resources); } } } return aggregatedResources; } private void generateStatistics() throws Exception { statisticsFile = new File(directory, "all-corrected.csv"); if(statisticsFile.exists()) { statisticsFile.delete(); } statisticsFile.createNewFile(); StatisticsCollector statisticsCollector = new StatisticsCollector(); Map contextCount = statisticsCollector.getContextCount(); addContextCount(contextCount); SortedMap contextStatistics = statisticsCollector.getStatistics(); SortedMap,Set>>> aggregatedResources = getAggregatedResources( contextStatistics); generateAggregatedStatistics(aggregatedResources); } @Test public void test() throws Exception { ContextTest.setContextByName("/d4science.research-infrastructures.eu"); generateStatistics(); } }