diff --git a/src/test/java/org/gcube/phd/ContextStatistic.java b/src/test/java/org/gcube/phd/ContextStatistic.java index 098a38a..0931e80 100644 --- a/src/test/java/org/gcube/phd/ContextStatistic.java +++ b/src/test/java/org/gcube/phd/ContextStatistic.java @@ -1,26 +1,60 @@ package org.gcube.phd; +import static org.gcube.common.authorization.client.Constants.authorizationService; + import java.io.UnsupportedEncodingException; import java.util.ArrayList; +import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedMap; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.gcube.accounting.aggregator.utility.Utility; +import org.gcube.accounting.analytics.Filter; +import org.gcube.accounting.analytics.Info; +import org.gcube.accounting.analytics.TemporalConstraint; +import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; +import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; +import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory; +import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; +import org.gcube.common.authorization.client.Constants; +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.authorization.library.provider.UserInfo; import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.HostingNode; import org.gcube.common.resources.gcore.Resource; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.Software; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.documentstore.records.AggregatedRecord; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.gcube.resources.discovery.icclient.ICFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ContextStatistic { + protected Logger logger = LoggerFactory.getLogger(ContextStatistic.class); + + public static Calendar START_TIME; + public static Calendar END_TIME; + private static AccountingPersistenceQuery apq; + + static { + START_TIME = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1); + END_TIME = Utility.getAggregationStartCalendar(2017, Calendar.DECEMBER, 1); + apq = AccountingPersistenceQueryFactory.getInstance(); + } + private ScopeBean scopeBean; public ScopeBean getScopeBean() { @@ -29,6 +63,7 @@ public class ContextStatistic { private Map,Set>> resourceMap; private Map,SummaryStatistics> statisticsMap; + private SortedMap accountingData; public ContextStatistic(ScopeBean scopeBean) { this.scopeBean = scopeBean; @@ -36,9 +71,13 @@ public class ContextStatistic { public void analizeInstances(Class clz) throws Exception { Set> resources = resourceMap.get(clz); - + if(resources==null) { + resources = new TreeSet<>(); + resourceMap.put(clz, resources); + } DiscoveryClient client = ICFactory.clientFor(clz); SimpleQuery query = ICFactory.queryFor(clz); + // logger.debug(query.expression()); List instances = client.submit(query); for(R r : instances) { ResourceInfo resourceInfo = new ResourceInfo(r); @@ -46,6 +85,17 @@ public class ContextStatistic { } } + @SuppressWarnings("unused") + private String generateUserToken(String context) throws Exception { + AuthorizationEntry authorizationEntry = Constants.authorizationService() + .get(SecurityTokenProvider.instance.get()); + UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo(); + String userToken = authorizationService().generateUserToken(userInfo, context); + logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken); + return userToken; + } + + public Map,Set>> getAllResources() throws Exception { if(resourceMap == null) { resourceMap = new HashMap<>(); @@ -56,7 +106,15 @@ public class ContextStatistic { classes.add(GenericResource.class); classes.add(Software.class); + String contextFullName = scopeBean.toString(); + /* + String token = generateUserToken(contextFullName); + ContextTest.setContext(token); + */ + ScopeProvider.instance.set(contextFullName); + for(Class clz : classes) { + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); analizeInstances(clz); } } @@ -73,11 +131,32 @@ public class ContextStatistic { for(ResourceInfo resourceInfo : resources) { summaryStatistics.addValue(resourceInfo.getSize()); } + statisticsMap.put(clz, summaryStatistics); } return statisticsMap; } - public Map,SummaryStatistics> getStatistcs() throws Exception { + public SortedMap getAccountingData() throws Exception { + if(accountingData==null) { + + TemporalConstraint temporalConstraint = new TemporalConstraint( + START_TIME.getTimeInMillis(), END_TIME.getTimeInMillis(), + AggregationMode.MONTHLY); + List filters = new ArrayList<>(); + Filter filter = new Filter("serviceClass", "InformationSystem"); + filters.add(filter); + + Class> clz = AggregatedServiceUsageRecord.class; + + // AccountingPersistenceQueryFactory.getForcedQueryScope().set(scopeBean.toString()); + + accountingData = apq.getTimeSeries(clz, temporalConstraint, + filters, true); + } + return accountingData; + } + + public Map,SummaryStatistics> getStatistics() throws Exception { if(statisticsMap == null) { getAllResources(); statisticsMap = generateStatistics(resourceMap); diff --git a/src/test/java/org/gcube/phd/ResourceInfo.java b/src/test/java/org/gcube/phd/ResourceInfo.java index 9c3bd51..d4f1ad1 100644 --- a/src/test/java/org/gcube/phd/ResourceInfo.java +++ b/src/test/java/org/gcube/phd/ResourceInfo.java @@ -6,7 +6,7 @@ import java.io.UnsupportedEncodingException; import org.gcube.common.resources.gcore.Resource; import org.gcube.common.resources.gcore.Resources; -public class ResourceInfo { +public class ResourceInfo implements Comparable> { private R r; private String unmarshalledResource; @@ -25,6 +25,10 @@ public class ResourceInfo { return unmarshalledResource; } + public String getResourceID() { + return r.id(); + } + public int getSize() throws UnsupportedEncodingException { if(size==null) { String unmarshalledR = getUnmarshalledResource(); @@ -44,7 +48,7 @@ public class ResourceInfo { return false; @SuppressWarnings("unchecked") ResourceInfo other = (ResourceInfo) obj; - return this.r.id().compareTo(other.r.id())==0; + return this.compareTo(other)==0; } @Override @@ -54,5 +58,10 @@ public class ResourceInfo { result = prime * r.id().hashCode(); return result; } + + @Override + public int compareTo(ResourceInfo other) { + return this.r.id().compareTo(other.r.id()); + } } diff --git a/src/test/java/org/gcube/phd/StatisticsCollector.java b/src/test/java/org/gcube/phd/StatisticsCollector.java index 268ce38..7366f70 100644 --- a/src/test/java/org/gcube/phd/StatisticsCollector.java +++ b/src/test/java/org/gcube/phd/StatisticsCollector.java @@ -1,17 +1,10 @@ package org.gcube.phd; -import static org.gcube.common.authorization.client.Constants.authorizationService; - import java.util.HashMap; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; -import org.gcube.common.authorization.client.Constants; -import org.gcube.common.authorization.library.AuthorizationEntry; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.authorization.library.provider.UserInfo; -import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.context.ContextElaborator; @@ -23,17 +16,6 @@ public class StatisticsCollector extends ContextElaborator { public StatisticsCollector() {} - @SuppressWarnings("unused") - private String generateUserToken(String context) throws Exception { - AuthorizationEntry authorizationEntry = Constants.authorizationService() - .get(SecurityTokenProvider.instance.get()); - UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo(); - String userToken = authorizationService().generateUserToken(userInfo, context); - logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken); - return userToken; - } - - @Override public void elaborateContext(ScopeBean scopeBean) throws Exception { Type type = scopeBean.type(); @@ -43,14 +25,6 @@ public class StatisticsCollector extends ContextElaborator { } contextCount.put(type, integer + 1); - - String contextFullName = scopeBean.toString(); - /* - String token = generateUserToken(contextFullName); - ContextTest.setContext(token); - */ - ScopeProvider.instance.set(contextFullName); - ContextStatistic contextStatistic = new ContextStatistic(scopeBean); sortedMap.put(scopeBean, contextStatistic); } diff --git a/src/test/java/org/gcube/phd/StatisticsGenerator.java b/src/test/java/org/gcube/phd/StatisticsGenerator.java index b0905e4..2c73864 100644 --- a/src/test/java/org/gcube/phd/StatisticsGenerator.java +++ b/src/test/java/org/gcube/phd/StatisticsGenerator.java @@ -5,6 +5,8 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import java.text.SimpleDateFormat; +import java.util.Calendar; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -13,19 +15,25 @@ import java.util.SortedMap; import java.util.TreeMap; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.gcube.accounting.analytics.Info; +import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.common.resources.gcore.Resource; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.testutility.ContextTest; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class StatisticsGenerator { + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + private File directory; private File statisticsFile; public StatisticsGenerator() { - directory = new File("/home/lucafrosini/Desktop/GlobalStatistics"); + directory = new File("/home/lucafrosini/Dropbox/GlobalStatistics"); if(!directory.exists()) { directory.mkdirs(); } @@ -51,7 +59,7 @@ public class StatisticsGenerator { stringBuffer.append("Count"); printLineToFile(stringBuffer.toString(), statisticsFile); - for(Type type : contextCount.keySet()) { + for(Type type : Type.values()) { stringBuffer = new StringBuffer(); stringBuffer.append(type.name()); stringBuffer.append(","); @@ -77,7 +85,10 @@ public class StatisticsGenerator { stringBuffer.append(","); stringBuffer.append("Min"); stringBuffer.append(","); - stringBuffer.append("Variance"); + stringBuffer.append("Standard Deviation"); + stringBuffer.append(","); + stringBuffer.append("Geometric Mean"); + stringBuffer.append(","); printLineToFile(stringBuffer.toString(), statisticsFile); } @@ -94,7 +105,10 @@ public class StatisticsGenerator { stringBuffer.append(","); stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue()); stringBuffer.append(","); - stringBuffer.append(Double.valueOf(summaryStatistics.getVariance()).intValue()); + stringBuffer.append(Double.valueOf(summaryStatistics.getStandardDeviation()).intValue()); + stringBuffer.append(","); + stringBuffer.append(Double.valueOf(summaryStatistics.getGeometricMean()).intValue()); + printLineToFile(stringBuffer.toString(), statisticsFile); } @@ -104,28 +118,73 @@ public class StatisticsGenerator { } } - private void generateStatistics( + private void contextStatisticReport(ScopeBean scopeBean, Map,SummaryStatistics> statistics) throws Exception { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append(scopeBean.toString()); + printLineToFile(stringBuffer.toString(), statisticsFile); + addHeader(); + + addStatistics(statistics); + } + + private void contextAccountingReport(ScopeBean scopeBean, SortedMap accountingData) throws Exception { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + simpleDateFormat.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE); + StringBuffer calendarBuffer = new StringBuffer(); + StringBuffer dataBuffer = new StringBuffer(); + boolean first = true; + for(Calendar calendar : accountingData.keySet()) { + if(first) { + first = false; + }else { + calendarBuffer.append(","); + dataBuffer.append(","); + } + calendarBuffer.append(simpleDateFormat.format(calendar.getTime())); + Info info = accountingData.get(calendar); + int operationCount = (int) info.getValue().get("operationCount"); + dataBuffer.append(operationCount); + } + printLineToFile(calendarBuffer.toString(), statisticsFile); + printLineToFile(dataBuffer.toString(), statisticsFile); + } + + private void generateAggregatedStatistics( SortedMap,Set>>> aggregatedResources) throws Exception { + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + printLineToFile("######################################", statisticsFile); + printLineToFile("Aggregated Statistics", statisticsFile); + for(ScopeBean scopeBean : aggregatedResources.keySet()) { - StringBuffer stringBuffer = new StringBuffer(); - stringBuffer.append(scopeBean.toString()); - stringBuffer.append(" Statistics"); - printLineToFile(stringBuffer.toString(), statisticsFile); - addHeader(); - Map,Set>> resourceMap = aggregatedResources .get(scopeBean); Map,SummaryStatistics> statistics = ContextStatistic .generateStatistics(resourceMap); - addStatistics(statistics); - printLineToFile("", statisticsFile); - printLineToFile("", statisticsFile); + contextStatisticReport(scopeBean, statistics); + } } + + private void generateContextStatisticReport(ScopeBean scopeBean, ContextStatistic contextStatistic) throws Exception { + contextStatisticReport(scopeBean, contextStatistic.getStatistics()); + /* + try { + contextAccountingReport(scopeBean, contextStatistic.getAccountingData()); + }catch (Exception e) { + logger.error("Unable to Generate AccountignData resport for contes {}", scopeBean.toString(), e); + printLineToFile("PARTIAL OR INVALID ACCOUNTING DATA", statisticsFile); + } + */ + printLineToFile("", statisticsFile); + printLineToFile("", statisticsFile); + } + private SortedMap,Set>>> getAggregatedResources( SortedMap contextStatistics) throws Exception { @@ -138,6 +197,11 @@ public class StatisticsGenerator { for(ScopeBean scopeBean : contextStatistics.keySet()) { ContextStatistic contextStatistic = contextStatistics.get(scopeBean); + generateContextStatisticReport(scopeBean, contextStatistic); + + + logger.debug("Generating AggregatedStatistics for {}", scopeBean.toString()); + Map,Set>> resourceMap = contextStatistic .getAllResources(); @@ -160,23 +224,45 @@ public class StatisticsGenerator { break; } + File dir = new File(directory, scopeBean.toString()); + if(!dir.exists()) { + dir.mkdirs(); + } + for(Class clz : resourceMap.keySet()) { Set> resources = resourceMap.get(clz); + /* + File resourceDir = new File(dir, "__Resources"); + resourceDir = new File(resourceDir, clz.getSimpleName()); + if(!resourceDir.exists()) { + resourceDir.mkdirs(); + } + for(ResourceInfo resourceInfo : resources) { + File f = new File(resourceDir, resourceInfo.getResourceID()); + if(f.exists()) { + f.delete(); + } + f.createNewFile(); + printLineToFile(resourceInfo.getUnmarshalledResource(),f); + } + */ + Set> infrastructureResources = infrastructureResourcesByClass.get(clz); if(infrastructureResources == null) { infrastructureResources = new HashSet<>(); infrastructureResourcesByClass.put(clz, infrastructureResources); } - infrastructureResources.addAll(resources); - Set> voResources = voResourcesByClass.get(clz); - if(voResources == null) { - voResources = new HashSet<>(); - voResourcesByClass.put(clz, voResources); + if(type != Type.INFRASTRUCTURE) { + Set> voResources = voResourcesByClass.get(clz); + if(voResources == null) { + voResources = new HashSet<>(); + voResourcesByClass.put(clz, voResources); + } + voResources.addAll(resources); } - voResources.addAll(resources); } @@ -186,12 +272,15 @@ public class StatisticsGenerator { } + + private void generateStatistics() throws Exception { - statisticsFile = new File(directory, "all.csv"); + statisticsFile = new File(directory, "all-corrected.csv"); if(statisticsFile.exists()) { statisticsFile.delete(); } statisticsFile.createNewFile(); + StatisticsCollector statisticsCollector = new StatisticsCollector(); Map contextCount = statisticsCollector.getContextCount(); addContextCount(contextCount); @@ -199,7 +288,7 @@ public class StatisticsGenerator { SortedMap contextStatistics = statisticsCollector.getStatistics(); SortedMap,Set>>> aggregatedResources = getAggregatedResources( contextStatistics); - generateStatistics(aggregatedResources); + generateAggregatedStatistics(aggregatedResources); } @Test