gCubeSystem
/
accounting-aggregator-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added test to analyse aggreation results

This commit is contained in:
Luca Frosini 2020-04-17 17:34:13 +02:00
parent bd213405d5
commit e9a3239f07
3 changed files with 125 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
src/test/java/org/gcube/accounting/aggregator/analysis/TestRules.java Normal file
View File

@ -0,0 +1,107 @@
package org.gcube.accounting.aggregator.analysis;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FilenameFilter;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.gcube.accounting.aggregator.utility.Utility;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestRules {
private static final Logger logger = LoggerFactory.getLogger(TestRules.class);
public File getCalledMethodCSVDirectory() throws Exception {
URL logbackFileURL = TestRules.class.getClassLoader().getResource("logback-test.xml");
File logbackFile = new File(logbackFileURL.toURI());
File resourcesDirectory = logbackFile.getParentFile();
return new File(resourcesDirectory, "CalledMethods");
}
public List<File> allCSV() throws Exception {
File calledMethodCSVDirectory = getCalledMethodCSVDirectory();
FilenameFilter filenameFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".csv");
}
};
List<File> csvFiles = Arrays.asList(calledMethodCSVDirectory.listFiles(filenameFilter));
return csvFiles;
}
public void analyseCSVFile(File csvFile, Map<String,Map<String, Integer>> map) throws Exception {
logger.info("Analisyng CSV file {}\n", csvFile.getAbsolutePath());
try(BufferedReader br = new BufferedReader(new FileReader(csvFile))) {
for(String line; (line = br.readLine()) != null;) {
String[] splittedLine = line.split(",");
String serviceClass = splittedLine[0].substring(1,splittedLine[0].length()-1);
String serviceName = splittedLine[1].substring(1,splittedLine[1].length()-1);
String calledMethod = splittedLine[2].substring(1,splittedLine[2].length()-1);
String countString = splittedLine[3].substring(1,splittedLine[3].length()-1);
Integer count = Integer.valueOf(countString);
String serviceClassServiceName = serviceClass + "," + serviceName;
Map<String, Integer> calledMethodCountMap = map.get(serviceClassServiceName);
if(calledMethodCountMap==null) {
calledMethodCountMap = new TreeMap<String, Integer>();
map.put(serviceClassServiceName, calledMethodCountMap);
}
Integer gotCount = calledMethodCountMap.get(calledMethod);
if(gotCount==null) {
gotCount = 0;
}
calledMethodCountMap.put(calledMethod, gotCount + count);
}
} catch(Exception e) {
throw e;
}
}
@Test
public void createUniqueCSV() throws Exception {
List<File> csvFiles = allCSV();
Map<String,Map<String, Integer>> map = new TreeMap<>();
for(File csvFile : csvFiles) {
analyseCSVFile(csvFile, map);
}
generateCSV(map);
}
public void generateCSV(Map<String,Map<String, Integer>> map) throws Exception {
File generatedCSV = new File("generated.csv");
logger.info("Going to generated CSV suammary file {}\n", generatedCSV.getAbsolutePath());
if(generatedCSV.exists()) {
generatedCSV.delete();
}
for(String serviceClassServiceName : map.keySet()) {
Map<String, Integer> calledMethodCountMap = map.get(serviceClassServiceName);
for(String calledMethod : calledMethodCountMap.keySet()) {
Integer count = calledMethodCountMap.get(calledMethod);
Utility.printLine(generatedCSV, "\"" + serviceClassServiceName.replace(",", "\",\"") + "\",\"" + calledMethod + "\"," + String.valueOf(count.intValue()));
}
}
logger.info("File {} successfully generated\n", generatedCSV.getAbsolutePath());
}
}

34
src/test/java/org/gcube/accounting/aggregator/plugin/AccountingAggregatorPluginTest.java
View File

@ -3,6 +3,7 @@ package org.gcube.accounting.aggregator.plugin;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.gcube.accounting.aggregator.aggregation.AggregationType;
import org.gcube.accounting.aggregator.plugin.AccountingAggregatorPlugin.ElaborationType;
@ -17,17 +18,17 @@ public class AccountingAggregatorPluginTest extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(AccountingAggregatorPluginTest.class);
// private static final String ROOT_PROD = "/d4science.research-infrastructures.eu";
public static final String ROOT_PROD = "/d4science.research-infrastructures.eu";
@Test
public void aggregate() throws Exception {
ContextTest.setContextByName(PARENT_DEFAULT_TEST_SCOPE);
// ContextTest.setContextByName(ROOT_PROD);
//ContextTest.setContextByName(ROOT_DEV_SCOPE);
ContextTest.setContextByName(ROOT_PROD);
Map<String, Object> inputs = new HashMap<String, Object>();
AggregationType aggregationType = AggregationType.YEARLY;
AggregationType aggregationType = AggregationType.MONTHLY;
//type aggregation
inputs.put(AccountingAggregatorPlugin.AGGREGATION_TYPE_INPUT_PARAMETER, aggregationType.name());
@ -46,13 +47,14 @@ public class AccountingAggregatorPluginTest extends ContextTest {
inputs.put(AccountingAggregatorPlugin.FORCE_RERUN, true);
inputs.put(AccountingAggregatorPlugin.FORCE_RESTART, true);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.JANUARY, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2018, Calendar.SEPTEMBER, 1);
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2018, Calendar.NOVEMBER, 1);
/*
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.DECEMBER, 31);
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
@ -61,12 +63,11 @@ public class AccountingAggregatorPluginTest extends ContextTest {
AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin(null);
logger.debug("Going to launch {} with inputs {}", AccountingAggregatorPluginDeclaration.NAME, inputs);
// Calendar end = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
Calendar end = Utility.getAggregationStartCalendar(2020, Calendar.JANUARY, 1);
while(aggregationStartCalendar.before(end)) {
while(aggregationStartCalendar.before(aggregationEndCalendar)) {
plugin.launch(inputs);
Thread.sleep(TimeUnit.MINUTES.toMillis(1));
aggregationStartCalendar.add(aggregationType.getCalendarField(), 1);
aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
@ -86,7 +87,7 @@ public class AccountingAggregatorPluginTest extends ContextTest {
@Test
public void testRecovery() throws Exception {
ContextTest.setContextByName(PARENT_DEFAULT_TEST_SCOPE);
ContextTest.setContextByName(ROOT_DEV_SCOPE);
// ContextTest.setContextByName(ROOT_PROD);
Map<String, Object> inputs = new HashMap<String, Object>();
@ -97,27 +98,26 @@ public class AccountingAggregatorPluginTest extends ContextTest {
inputs.put(AccountingAggregatorPlugin.PERSIST_START_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(8, 0));
inputs.put(AccountingAggregatorPlugin.PERSIST_END_TIME_INPUT_PARAMETER, Utility.getPersistTimeParameter(20, 30));
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2017, Calendar.DECEMBER, 1);
Calendar aggregationStartCalendar = Utility.getAggregationStartCalendar(2020, Calendar.MARCH, 1);
String aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationStartDate);
inputs.put(AccountingAggregatorPlugin.FORCE_EARLY_AGGREGATION, true);
/*
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2017, Calendar.SEPTEMBER, 30);
// Calendar aggregationEndCalendar = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
Calendar aggregationEndCalendar = Utility.getAggregationStartCalendar(2020, Calendar.APRIL, 1);
String aggregationEndDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationEndCalendar.getTime());
logger.trace("{} : {}", AccountingAggregatorPlugin.AGGREGATION_START_DATE_INPUT_PARAMETER, aggregationEndDate);
inputs.put(AccountingAggregatorPlugin.AGGREGATION_END_DATE_INPUT_PARAMETER, aggregationEndDate);
*/
AccountingAggregatorPlugin plugin = new AccountingAggregatorPlugin(null);
logger.debug("Going to launch {} with inputs {}", AccountingAggregatorPluginDeclaration.NAME, inputs);
// Calendar end = Utility.getEndCalendarFromStartCalendar(AggregationType.MONTHLY, aggregationStartCalendar, 1);
//Calendar end = Utility.getAggregationStartCalendar(2018, Calendar.JANUARY, 1);
/*
while(aggregationStartCalendar.before(end)) {
while(aggregationStartCalendar.before(aggregationEndCalendar)) {
plugin.launch(inputs);
aggregationStartCalendar.add(Calendar.MONTH, 1);
aggregationStartDate = AccountingAggregatorPlugin.AGGREGATION_START_DATE_DATE_FORMAT.format(aggregationStartCalendar.getTime());

1
src/test/resources/.gitignore vendored
View File

@ -4,3 +4,4 @@
/preprod.gcubekey
/token.properties
/d4science.research-infrastructures.eu.gcubekey
/CalledMethods/