Added utility

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/private/luca.frosini/infrastructure-tests@178459 82a268e6-3cf1-43bd-a215-b396298e98cf

src/test/java/org/gcube/phd/ContextStatistic.java

@@ -1,26 +1,60 @@
 package org.gcube.phd;
 
+import static org.gcube.common.authorization.client.Constants.authorizationService;
+
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.gcube.accounting.aggregator.utility.Utility;
+import org.gcube.accounting.analytics.Filter;
+import org.gcube.accounting.analytics.Info;
+import org.gcube.accounting.analytics.TemporalConstraint;
+import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
+import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
+import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
+import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
+import org.gcube.common.authorization.client.Constants;
+import org.gcube.common.authorization.library.AuthorizationEntry;
+import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
+import org.gcube.common.authorization.library.provider.UserInfo;
 import org.gcube.common.resources.gcore.GCoreEndpoint;
 import org.gcube.common.resources.gcore.GenericResource;
 import org.gcube.common.resources.gcore.HostingNode;
 import org.gcube.common.resources.gcore.Resource;
 import org.gcube.common.resources.gcore.ServiceEndpoint;
 import org.gcube.common.resources.gcore.Software;
+import org.gcube.common.scope.api.ScopeProvider;
 import org.gcube.common.scope.impl.ScopeBean;
+import org.gcube.documentstore.records.AggregatedRecord;
 import org.gcube.resources.discovery.client.api.DiscoveryClient;
 import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
 import org.gcube.resources.discovery.icclient.ICFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ContextStatistic {
 	
+	protected Logger logger = LoggerFactory.getLogger(ContextStatistic.class);
+	
+	public static Calendar START_TIME;
+	public static Calendar END_TIME;
+	private static AccountingPersistenceQuery apq;
+	
+	static {
+		START_TIME = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1);
+		END_TIME = Utility.getAggregationStartCalendar(2017, Calendar.DECEMBER, 1);
+		apq = AccountingPersistenceQueryFactory.getInstance();
+	}
+	
 	private ScopeBean scopeBean;
 	
 	public ScopeBean getScopeBean() {
@@ -29,6 +63,7 @@ public class ContextStatistic {
 	
 	private Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap;
 	private Map<Class<? extends Resource>,SummaryStatistics> statisticsMap;
+	private SortedMap<Calendar, Info> accountingData;
 	
 	public ContextStatistic(ScopeBean scopeBean) {
 		this.scopeBean = scopeBean;
@@ -36,9 +71,13 @@ public class ContextStatistic {
 	
 	public <R extends Resource> void analizeInstances(Class<R> clz) throws Exception {
 		Set<ResourceInfo<? extends Resource>> resources = resourceMap.get(clz);
-		
+		if(resources==null) {
+			resources = new TreeSet<>();
+			resourceMap.put(clz, resources);
+		}
 		DiscoveryClient<R> client = ICFactory.clientFor(clz);
 		SimpleQuery query = ICFactory.queryFor(clz);
+		// logger.debug(query.expression());
 		List<R> instances = client.submit(query);
 		for(R r : instances) {
 			ResourceInfo<R> resourceInfo = new ResourceInfo<R>(r);
@@ -46,6 +85,17 @@ public class ContextStatistic {
 		}
 	}
 	
+	@SuppressWarnings("unused")
+	private String generateUserToken(String context) throws Exception {
+		AuthorizationEntry authorizationEntry = Constants.authorizationService()
+				.get(SecurityTokenProvider.instance.get());
+		UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo();
+		String userToken = authorizationService().generateUserToken(userInfo, context);
+		logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken);
+		return userToken;
+	}
+	
+	
 	public Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> getAllResources() throws Exception {
 		if(resourceMap == null) {
 			resourceMap = new HashMap<>();
@@ -56,7 +106,15 @@ public class ContextStatistic {
 			classes.add(GenericResource.class);
 			classes.add(Software.class);
 			
+			String contextFullName = scopeBean.toString();
+			/*
+			String token = generateUserToken(contextFullName);
+			ContextTest.setContext(token);
+			*/
+			ScopeProvider.instance.set(contextFullName);
+			
 			for(Class<? extends Resource> clz : classes) {
+				Thread.sleep(TimeUnit.SECONDS.toMillis(5));
 				analizeInstances(clz);
 			}
 		}
@@ -73,11 +131,32 @@ public class ContextStatistic {
 			for(ResourceInfo<? extends Resource> resourceInfo : resources) {
 				summaryStatistics.addValue(resourceInfo.getSize());
 			}
+			statisticsMap.put(clz, summaryStatistics);
 		}
 		return statisticsMap;
 	}
 	
-	public Map<Class<? extends Resource>,SummaryStatistics> getStatistcs() throws Exception {
+	public SortedMap<Calendar, Info> getAccountingData() throws Exception {
+		if(accountingData==null) {
+		
+			TemporalConstraint temporalConstraint = new TemporalConstraint(
+					START_TIME.getTimeInMillis(), END_TIME.getTimeInMillis(),
+					AggregationMode.MONTHLY);
+			List<Filter> filters = new ArrayList<>();
+			Filter filter = new Filter("serviceClass", "InformationSystem");
+			filters.add(filter);
+			
+			Class<? extends AggregatedRecord<?, ?>> clz = AggregatedServiceUsageRecord.class;
+			
+			// AccountingPersistenceQueryFactory.getForcedQueryScope().set(scopeBean.toString());
+			
+			accountingData = apq.getTimeSeries(clz, temporalConstraint,
+					filters, true);
+		}
+		return accountingData;
+	}
+	
+	public Map<Class<? extends Resource>,SummaryStatistics> getStatistics() throws Exception {
 		if(statisticsMap == null) {
 			getAllResources();
 			statisticsMap = generateStatistics(resourceMap);

src/test/java/org/gcube/phd/ResourceInfo.java

@@ -6,7 +6,7 @@ import java.io.UnsupportedEncodingException;
 import org.gcube.common.resources.gcore.Resource;
 import org.gcube.common.resources.gcore.Resources;
 
-public class ResourceInfo<R extends Resource> {
+public class ResourceInfo<R extends Resource> implements Comparable<ResourceInfo<R>> {
 	
 	private R r;
 	private String unmarshalledResource;
@@ -25,6 +25,10 @@ public class ResourceInfo<R extends Resource> {
 		return unmarshalledResource;
 	}
 	
+	public String getResourceID() {
+		return r.id();
+	}
+	
 	public int getSize() throws UnsupportedEncodingException {
 		if(size==null) {
 			String unmarshalledR = getUnmarshalledResource();
@@ -44,7 +48,7 @@ public class ResourceInfo<R extends Resource> {
 	        return false;
 	    @SuppressWarnings("unchecked")
 		ResourceInfo<R> other = (ResourceInfo<R>) obj;
-	    return this.r.id().compareTo(other.r.id())==0;
+	    return this.compareTo(other)==0;
 	}
 
 	@Override
@@ -54,5 +58,10 @@ public class ResourceInfo<R extends Resource> {
 	    result = prime * r.id().hashCode();
 	    return result;
 	}
+
+	@Override
+	public int compareTo(ResourceInfo<R> other) {
+		return this.r.id().compareTo(other.r.id());
+	}
 	
 }

src/test/java/org/gcube/phd/StatisticsCollector.java

@@ -1,17 +1,10 @@
 package org.gcube.phd;
 
-import static org.gcube.common.authorization.client.Constants.authorizationService;
-
 import java.util.HashMap;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-import org.gcube.common.authorization.client.Constants;
-import org.gcube.common.authorization.library.AuthorizationEntry;
-import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
-import org.gcube.common.authorization.library.provider.UserInfo;
-import org.gcube.common.scope.api.ScopeProvider;
 import org.gcube.common.scope.impl.ScopeBean;
 import org.gcube.common.scope.impl.ScopeBean.Type;
 import org.gcube.context.ContextElaborator;
@@ -23,17 +16,6 @@ public class StatisticsCollector extends ContextElaborator {
 	
 	public StatisticsCollector() {}
 	
-	@SuppressWarnings("unused")
-	private String generateUserToken(String context) throws Exception {
-		AuthorizationEntry authorizationEntry = Constants.authorizationService()
-				.get(SecurityTokenProvider.instance.get());
-		UserInfo userInfo = (UserInfo) authorizationEntry.getClientInfo();
-		String userToken = authorizationService().generateUserToken(userInfo, context);
-		logger.trace("Token for Context {} for {} is {}", context, userInfo.getId(), userToken);
-		return userToken;
-	}
-	
-	
 	@Override
 	public void elaborateContext(ScopeBean scopeBean) throws Exception {
 		Type type = scopeBean.type();
@@ -43,14 +25,6 @@ public class StatisticsCollector extends ContextElaborator {
 		}
 		contextCount.put(type, integer + 1);
 		
-		
-		String contextFullName = scopeBean.toString();
-		/*
-		String token = generateUserToken(contextFullName);
-		ContextTest.setContext(token);
-		*/
-		ScopeProvider.instance.set(contextFullName);
-		
 		ContextStatistic contextStatistic = new ContextStatistic(scopeBean);
 		sortedMap.put(scopeBean, contextStatistic);
 	}

src/test/java/org/gcube/phd/StatisticsGenerator.java

@@ -5,6 +5,8 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -13,19 +15,25 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 
 import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.gcube.accounting.analytics.Info;
+import org.gcube.accounting.analytics.TemporalConstraint;
 import org.gcube.common.resources.gcore.Resource;
 import org.gcube.common.scope.impl.ScopeBean;
 import org.gcube.common.scope.impl.ScopeBean.Type;
 import org.gcube.testutility.ContextTest;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class StatisticsGenerator {
 	
+	protected Logger logger = LoggerFactory.getLogger(this.getClass());
+	
 	private File directory;
 	private File statisticsFile;
 	
 	public StatisticsGenerator() {
-		directory = new File("/home/lucafrosini/Desktop/GlobalStatistics");
+		directory = new File("/home/lucafrosini/Dropbox/GlobalStatistics");
 		if(!directory.exists()) {
 			directory.mkdirs();
 		}
@@ -51,7 +59,7 @@ public class StatisticsGenerator {
 		stringBuffer.append("Count");
 		printLineToFile(stringBuffer.toString(), statisticsFile);
 		
-		for(Type type : contextCount.keySet()) {
+		for(Type type : Type.values()) {
 			stringBuffer = new StringBuffer();
 			stringBuffer.append(type.name());
 			stringBuffer.append(",");
@@ -77,7 +85,10 @@ public class StatisticsGenerator {
 		stringBuffer.append(",");
 		stringBuffer.append("Min");
 		stringBuffer.append(",");
-		stringBuffer.append("Variance");
+		stringBuffer.append("Standard Deviation");
+		stringBuffer.append(",");
+		stringBuffer.append("Geometric Mean");
+		stringBuffer.append(",");
 		printLineToFile(stringBuffer.toString(), statisticsFile);
 	}
 	
@@ -94,7 +105,10 @@ public class StatisticsGenerator {
 		stringBuffer.append(",");
 		stringBuffer.append(Double.valueOf(summaryStatistics.getMin()).intValue());
 		stringBuffer.append(",");
-		stringBuffer.append(Double.valueOf(summaryStatistics.getVariance()).intValue());
+		stringBuffer.append(Double.valueOf(summaryStatistics.getStandardDeviation()).intValue());
+		stringBuffer.append(",");
+		stringBuffer.append(Double.valueOf(summaryStatistics.getGeometricMean()).intValue());
+		
 		printLineToFile(stringBuffer.toString(), statisticsFile);
 	}
 	
@@ -104,28 +118,73 @@ public class StatisticsGenerator {
 		}
 	}
 	
-	private void generateStatistics(
+	private void contextStatisticReport(ScopeBean scopeBean, Map<Class<? extends Resource>,SummaryStatistics> statistics) throws Exception {
+		StringBuffer stringBuffer = new StringBuffer();
+		stringBuffer.append(scopeBean.toString());
+		printLineToFile(stringBuffer.toString(), statisticsFile);
+		addHeader();
+		
+		addStatistics(statistics);
+	}
+	
+	private void contextAccountingReport(ScopeBean scopeBean, SortedMap<Calendar,Info> accountingData) throws Exception {
+		SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+		simpleDateFormat.setTimeZone(TemporalConstraint.DEFAULT_TIME_ZONE);
+		StringBuffer calendarBuffer = new StringBuffer();
+		StringBuffer dataBuffer = new StringBuffer();
+		boolean first = true;
+		for(Calendar calendar : accountingData.keySet()) {
+			if(first) {
+				first = false;
+			}else {
+				calendarBuffer.append(",");
+				dataBuffer.append(",");
+			}
+			calendarBuffer.append(simpleDateFormat.format(calendar.getTime()));
+			Info info = accountingData.get(calendar);
+			int operationCount = (int) info.getValue().get("operationCount");
+			dataBuffer.append(operationCount);
+		}
+		printLineToFile(calendarBuffer.toString(), statisticsFile);
+		printLineToFile(dataBuffer.toString(), statisticsFile);
+	}
+	
+	private void generateAggregatedStatistics(
 			SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources)
 			throws Exception {
 		
+		printLineToFile("", statisticsFile);
+		printLineToFile("", statisticsFile);
+		printLineToFile("", statisticsFile);
+		printLineToFile("######################################", statisticsFile);
+		printLineToFile("Aggregated Statistics", statisticsFile);
+		
 		for(ScopeBean scopeBean : aggregatedResources.keySet()) {
-			StringBuffer stringBuffer = new StringBuffer();
-			stringBuffer.append(scopeBean.toString());
-			stringBuffer.append(" Statistics");
-			printLineToFile(stringBuffer.toString(), statisticsFile);
-			addHeader();
-			
 			Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap = aggregatedResources
 					.get(scopeBean);
 			Map<Class<? extends Resource>,SummaryStatistics> statistics = ContextStatistic
 					.generateStatistics(resourceMap);
-			addStatistics(statistics);
 			
-			printLineToFile("", statisticsFile);
+			contextStatisticReport(scopeBean, statistics);
-			printLineToFile("", statisticsFile);
+			
 		}
 	}
 	
+	
+	private void generateContextStatisticReport(ScopeBean scopeBean, ContextStatistic contextStatistic) throws Exception {
+		contextStatisticReport(scopeBean, contextStatistic.getStatistics());
+		/*
+		try {
+			contextAccountingReport(scopeBean, contextStatistic.getAccountingData());
+		}catch (Exception e) {
+			logger.error("Unable to Generate AccountignData resport for contes {}", scopeBean.toString(), e);
+			printLineToFile("PARTIAL OR INVALID ACCOUNTING DATA", statisticsFile);
+		}
+		*/
+		printLineToFile("", statisticsFile);
+		printLineToFile("", statisticsFile);
+	}
+	
 	private SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> getAggregatedResources(
 			SortedMap<ScopeBean,ContextStatistic> contextStatistics) throws Exception {
 		
@@ -138,6 +197,11 @@ public class StatisticsGenerator {
 		for(ScopeBean scopeBean : contextStatistics.keySet()) {
 			
 			ContextStatistic contextStatistic = contextStatistics.get(scopeBean);
+			generateContextStatisticReport(scopeBean, contextStatistic);
+			
+			
+			logger.debug("Generating AggregatedStatistics for {}", scopeBean.toString());
+			
 			Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>> resourceMap = contextStatistic
 					.getAllResources();
 			
@@ -160,23 +224,45 @@ public class StatisticsGenerator {
 					break;
 			}
 			
+			File dir = new File(directory, scopeBean.toString());
+			if(!dir.exists()) {
+				dir.mkdirs();
+			}
+			
 			for(Class<? extends Resource> clz : resourceMap.keySet()) {
 				Set<ResourceInfo<? extends Resource>> resources = resourceMap.get(clz);
+				/*
+				File resourceDir = new File(dir, "__Resources");
+				resourceDir = new File(resourceDir, clz.getSimpleName());
+				if(!resourceDir.exists()) {
+					resourceDir.mkdirs();
+				}
+				for(ResourceInfo<? extends Resource> resourceInfo : resources) {
+					File f = new File(resourceDir, resourceInfo.getResourceID());
+					if(f.exists()) {
+						f.delete();
+					}
+					f.createNewFile();
+					printLineToFile(resourceInfo.getUnmarshalledResource(),f);
+				}
+				*/
+				
 				
 				Set<ResourceInfo<? extends Resource>> infrastructureResources = infrastructureResourcesByClass.get(clz);
 				if(infrastructureResources == null) {
 					infrastructureResources = new HashSet<>();
 					infrastructureResourcesByClass.put(clz, infrastructureResources);
 				}
-				
 				infrastructureResources.addAll(resources);
 				
-				Set<ResourceInfo<? extends Resource>> voResources = voResourcesByClass.get(clz);
+				if(type != Type.INFRASTRUCTURE) {
-				if(voResources == null) {
+					Set<ResourceInfo<? extends Resource>> voResources = voResourcesByClass.get(clz);
-					voResources = new HashSet<>();
+					if(voResources == null) {
-					voResourcesByClass.put(clz, voResources);
+						voResources = new HashSet<>();
+						voResourcesByClass.put(clz, voResources);
+					}
+					voResources.addAll(resources);
 				}
-				voResources.addAll(resources);
 				
 			}
 			
@@ -186,12 +272,15 @@ public class StatisticsGenerator {
 		
 	}
 	
+	
+
 	private void generateStatistics() throws Exception {
-		statisticsFile = new File(directory, "all.csv");
+		statisticsFile = new File(directory, "all-corrected.csv");
 		if(statisticsFile.exists()) {
 			statisticsFile.delete();
 		}
 		statisticsFile.createNewFile();
+		
 		StatisticsCollector statisticsCollector = new StatisticsCollector();
 		Map<Type,Integer> contextCount = statisticsCollector.getContextCount();
 		addContextCount(contextCount);
@@ -199,7 +288,7 @@ public class StatisticsGenerator {
 		SortedMap<ScopeBean,ContextStatistic> contextStatistics = statisticsCollector.getStatistics();
 		SortedMap<ScopeBean,Map<Class<? extends Resource>,Set<ResourceInfo<? extends Resource>>>> aggregatedResources = getAggregatedResources(
 				contextStatistics);
-		generateStatistics(aggregatedResources);
+		generateAggregatedStatistics(aggregatedResources);
 	}
 	
 	@Test