package org.gcube.accounting.datamodel.validations.validators; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.FilenameFilter; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.gcube.accounting.aggregator.RegexRulesAggregator; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.documentstore.exception.NotAggregatableRecordsExceptions; import org.gcube.documentstore.records.DSMapper; import org.gcube.testutility.ContextTest; import org.gcube.testutility.TestUsageRecord; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; public class TestRules extends ContextTest { private static final Logger logger = LoggerFactory.getLogger(TestRules.class); public File getRulesDirectory() throws Exception { URL logbackFileURL = TestRules.class.getClassLoader().getResource("logback-test.xml"); File logbackFile = new File(logbackFileURL.toURI()); File resourcesDirectory = logbackFile.getParentFile(); return new File(resourcesDirectory, "rules"); } protected long durationWeightedAverage(int numberA, long durationA, int numberB, long durationB) { return ((numberA * durationA) + (numberB * durationB)) / (numberA + numberB); } public List allRules() throws Exception { File rulesDirectory = getRulesDirectory(); FilenameFilter filenameFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.endsWith(".json"); } }; List rulesFiles = new ArrayList<>(); for(File dir : rulesDirectory.listFiles()) { if(!dir.isDirectory()) { continue; } rulesFiles.addAll(Arrays.asList(dir.listFiles(filenameFilter))); RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance(); for(File rulesFile : rulesFiles) { ObjectMapper mapper = DSMapper.getObjectMapper(); MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class); regexRulesAggregator.addMatcherReplace(matcherReplace); } } return rulesFiles; } public void testRule(File rulesFile) throws Exception { logger.info( "-----------------------------------------------------------------------------------------------------"); logger.info("Analisyng rule from file {}\n", rulesFile.getAbsolutePath()); File rulesDirectory = rulesFile.getParentFile(); ObjectMapper mapper = DSMapper.getObjectMapper(); MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class); Replace replacementRegex = matcherReplace.getReplacementRegex(); Map aggregatedMap = new HashMap<>(); AggregatedServiceUsageRecord aggregated; // it indicates that the rules can create multiple calledMethod because the replacement is based on // Named Capture Group final String requiredMatchesFileName = rulesFile.getName().replaceAll(".json", ".csv"); File elaborationFile = new File(rulesDirectory, requiredMatchesFileName); try(BufferedReader br = new BufferedReader(new FileReader(elaborationFile))) { for(String line; (line = br.readLine()) != null;) { String[] splittedLine = line.split(","); String serviceClass = splittedLine[0]; String serviceName = splittedLine[1]; String calledMethod = splittedLine[2]; Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod); if(replace != null) { logger.info("{} --> {},{},{}", line, replace.getServiceClass(), replace.getServiceName(), replace.getCalledMethod()); } else { logger.error("{} does not match {}. This MUST not occur.", line, matcherReplace.getMultiMatcher().toString()); throw new Exception(); } String obtainedTriple = replace.getServiceClass() + "+++" + replace.getServiceName() + "+++" + replace.getCalledMethod(); if(aggregatedMap.containsKey(obtainedTriple)) { aggregated = aggregatedMap.get(obtainedTriple); } else { ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord(); serviceUsageRecord.setServiceClass(replace.getServiceClass()); serviceUsageRecord.setServiceName(replace.getServiceName()); serviceUsageRecord.setCalledMethod(replace.getCalledMethod()); serviceUsageRecord.validate(); aggregated = new AggregatedServiceUsageRecord(serviceUsageRecord); aggregatedMap.put(obtainedTriple, aggregated); } ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord(); sur.setServiceClass(serviceClass); sur.setServiceName(serviceName); sur.setCalledMethod(calledMethod); sur.validate(); // logger.trace("To Be aggregated ServiceUsageRecord {}", sur); long oldMinInvocationTime = aggregated.getMinInvocationTime(); long oldMaxInvocationTime = aggregated.getMaxInvocationTime(); int oldOperationCount = aggregated.getOperationCount(); long oldDuration = aggregated.getDuration(); long surDuration = sur.getDuration(); aggregated.aggregate(sur); //logger.debug("Resulting Aggregated ServiceUsageRecord: {}", aggregated); long avgDuration = durationWeightedAverage(oldOperationCount, oldDuration, 1, surDuration); Assert.assertTrue(aggregated.getDuration() == (avgDuration)); Assert.assertTrue(aggregated.getOperationCount() == ++oldOperationCount); if(oldMinInvocationTime >= surDuration) { Assert.assertTrue(aggregated.getMinInvocationTime() == surDuration); } else { Assert.assertTrue(aggregated.getMinInvocationTime() == oldMinInvocationTime); } if(oldMaxInvocationTime >= surDuration) { Assert.assertTrue(aggregated.getMaxInvocationTime() == oldMaxInvocationTime); } else { Assert.assertTrue(aggregated.getMaxInvocationTime() == surDuration); } } } catch(Exception e) { throw e; } FilenameFilter filenameFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { boolean accept = name.endsWith(".csv"); return name.compareTo(requiredMatchesFileName) != 0 && accept; } }; for(File dir : rulesDirectory.listFiles()) { if(!dir.isDirectory()) { continue; } File[] elaborationFilesNoMatch = dir.listFiles(filenameFilter); for(File elaborationFileNoMatch : elaborationFilesNoMatch) { logger.info("Comparing examples which must not match from file {}", elaborationFileNoMatch.getName()); try(BufferedReader br = new BufferedReader(new FileReader(elaborationFileNoMatch))) { for(String line; (line = br.readLine()) != null;) { String[] splittedLine = line.split(","); String serviceClass = splittedLine[0]; String serviceName = splittedLine[1]; String calledMethod = splittedLine[2]; Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod); if(replace != null) { logger.error("{} match {} but it should NOT. This MUST not occur.", line, matcherReplace.getMultiMatcher().toString()); throw new Exception(); } else { logger.trace("{} does NOT match as requested", line, replacementRegex.getServiceClass(), replacementRegex.getServiceName(), replacementRegex.getCalledMethod()); } ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord(); sur.setServiceClass(serviceClass); sur.setServiceName(serviceName); sur.setCalledMethod(calledMethod); sur.validate(); //logger.trace("Should not be aggregated ServiceUsageRecord {}", sur); for(AggregatedServiceUsageRecord asur : aggregatedMap.values()) { try { asur.aggregate(sur); throw new Exception("The record has been aggregated and it should NOT"); } catch(NotAggregatableRecordsExceptions e) { //logger.trace("{} is not aggragable as expected", sur); } } } } catch(Exception e) { throw e; } } } logger.info( "-----------------------------------------------------------------------------------------------------\n\n\n"); } @Test public void testSingleRule() throws Exception { allRules(); File rulesDirectory = getRulesDirectory(); File rulesDirFile = new File(rulesDirectory, "Thredds"); File rulesFile = new File(rulesDirFile, "Thredds_METADATA::OTHER.json"); testRule(rulesFile); } @Test public void testAllRules() throws Exception { List rulesFiles = allRules(); for(File rulesFile : rulesFiles) { testRule(rulesFile); } } }