2019-11-15 14:40:11 +01:00
|
|
|
package org.gcube.accounting.datamodel.validations.validators;
|
|
|
|
|
|
|
|
import java.io.BufferedReader;
|
|
|
|
import java.io.File;
|
|
|
|
import java.io.FileReader;
|
2019-11-25 17:33:44 +01:00
|
|
|
import java.io.FilenameFilter;
|
2019-11-15 14:40:11 +01:00
|
|
|
import java.net.URL;
|
2020-01-07 11:07:06 +01:00
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Arrays;
|
|
|
|
import java.util.HashMap;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
2019-11-15 14:40:11 +01:00
|
|
|
|
2019-11-26 11:50:31 +01:00
|
|
|
import org.gcube.accounting.aggregator.RegexRulesAggregator;
|
|
|
|
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
|
|
|
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
|
|
|
import org.gcube.documentstore.exception.NotAggregatableRecordsExceptions;
|
2019-11-15 14:40:11 +01:00
|
|
|
import org.gcube.documentstore.records.DSMapper;
|
2019-11-26 11:50:31 +01:00
|
|
|
import org.gcube.testutility.ContextTest;
|
|
|
|
import org.gcube.testutility.TestUsageRecord;
|
|
|
|
import org.junit.Assert;
|
2019-11-15 14:40:11 +01:00
|
|
|
import org.junit.Test;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
public class TestRules extends ContextTest {
|
2019-11-15 14:40:11 +01:00
|
|
|
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(TestRules.class);
|
|
|
|
|
2019-11-15 17:00:14 +01:00
|
|
|
public File getRulesDirectory() throws Exception {
|
|
|
|
URL logbackFileURL = TestRules.class.getClassLoader().getResource("logback-test.xml");
|
|
|
|
File logbackFile = new File(logbackFileURL.toURI());
|
|
|
|
File resourcesDirectory = logbackFile.getParentFile();
|
|
|
|
return new File(resourcesDirectory, "rules");
|
|
|
|
}
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
protected long durationWeightedAverage(int numberA, long durationA, int numberB, long durationB) {
|
2019-11-26 11:50:31 +01:00
|
|
|
return ((numberA * durationA) + (numberB * durationB)) / (numberA + numberB);
|
|
|
|
}
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
public List<File> allRules() throws Exception {
|
2019-11-26 11:50:31 +01:00
|
|
|
File rulesDirectory = getRulesDirectory();
|
|
|
|
FilenameFilter filenameFilter = new FilenameFilter() {
|
|
|
|
@Override
|
|
|
|
public boolean accept(File dir, String name) {
|
|
|
|
return name.endsWith(".json");
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
List<File> rulesFiles = new ArrayList<>();
|
|
|
|
for(File dir : rulesDirectory.listFiles()) {
|
|
|
|
if(!dir.isDirectory()) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
rulesFiles.addAll(Arrays.asList(dir.listFiles(filenameFilter)));
|
|
|
|
RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
|
2020-03-31 16:16:05 +02:00
|
|
|
regexRulesAggregator.vacumRules();
|
2020-01-07 11:07:06 +01:00
|
|
|
for(File rulesFile : rulesFiles) {
|
|
|
|
ObjectMapper mapper = DSMapper.getObjectMapper();
|
|
|
|
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
|
|
|
|
regexRulesAggregator.addMatcherReplace(matcherReplace);
|
|
|
|
}
|
|
|
|
|
2019-11-26 11:50:31 +01:00
|
|
|
}
|
2020-01-07 11:07:06 +01:00
|
|
|
|
|
|
|
return rulesFiles;
|
2019-11-26 11:50:31 +01:00
|
|
|
}
|
|
|
|
|
2019-11-25 17:33:44 +01:00
|
|
|
public void testRule(File rulesFile) throws Exception {
|
2020-01-07 11:07:06 +01:00
|
|
|
logger.info(
|
|
|
|
"-----------------------------------------------------------------------------------------------------");
|
2019-11-25 17:33:44 +01:00
|
|
|
logger.info("Analisyng rule from file {}\n", rulesFile.getAbsolutePath());
|
|
|
|
File rulesDirectory = rulesFile.getParentFile();
|
2019-11-15 17:00:14 +01:00
|
|
|
|
2019-11-15 14:40:11 +01:00
|
|
|
ObjectMapper mapper = DSMapper.getObjectMapper();
|
2019-11-15 17:00:14 +01:00
|
|
|
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
|
2020-01-07 11:07:06 +01:00
|
|
|
Replace replacementRegex = matcherReplace.getReplacementRegex();
|
2019-11-15 14:40:11 +01:00
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
Map<String,AggregatedServiceUsageRecord> aggregatedMap = new HashMap<>();
|
2019-11-26 11:50:31 +01:00
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
AggregatedServiceUsageRecord aggregated;
|
|
|
|
|
|
|
|
// it indicates that the rules can create multiple calledMethod because the replacement is based on
|
|
|
|
// Named Capture Group
|
2019-11-26 11:50:31 +01:00
|
|
|
|
2019-11-25 17:33:44 +01:00
|
|
|
final String requiredMatchesFileName = rulesFile.getName().replaceAll(".json", ".csv");
|
2020-01-07 11:07:06 +01:00
|
|
|
File elaborationFile = new File(rulesDirectory, requiredMatchesFileName);
|
2019-11-15 14:40:11 +01:00
|
|
|
try(BufferedReader br = new BufferedReader(new FileReader(elaborationFile))) {
|
|
|
|
for(String line; (line = br.readLine()) != null;) {
|
|
|
|
String[] splittedLine = line.split(",");
|
2019-11-26 11:50:31 +01:00
|
|
|
|
|
|
|
String serviceClass = splittedLine[0];
|
|
|
|
String serviceName = splittedLine[1];
|
|
|
|
String calledMethod = splittedLine[2];
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod);
|
|
|
|
if(replace != null) {
|
|
|
|
logger.info("{} --> {},{},{}", line, replace.getServiceClass(), replace.getServiceName(),
|
|
|
|
replace.getCalledMethod());
|
2019-11-15 14:40:11 +01:00
|
|
|
} else {
|
2020-01-07 11:07:06 +01:00
|
|
|
logger.error("{} does not match {}. This MUST not occur.", line,
|
|
|
|
matcherReplace.getMultiMatcher().toString());
|
2019-11-15 16:42:11 +01:00
|
|
|
throw new Exception();
|
2019-11-15 14:40:11 +01:00
|
|
|
}
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
String obtainedTriple = replace.getServiceClass() + "+++" + replace.getServiceName() + "+++"
|
|
|
|
+ replace.getCalledMethod();
|
|
|
|
if(aggregatedMap.containsKey(obtainedTriple)) {
|
|
|
|
aggregated = aggregatedMap.get(obtainedTriple);
|
|
|
|
} else {
|
|
|
|
ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord();
|
|
|
|
serviceUsageRecord.setServiceClass(replace.getServiceClass());
|
|
|
|
serviceUsageRecord.setServiceName(replace.getServiceName());
|
|
|
|
serviceUsageRecord.setCalledMethod(replace.getCalledMethod());
|
|
|
|
serviceUsageRecord.validate();
|
|
|
|
aggregated = new AggregatedServiceUsageRecord(serviceUsageRecord);
|
|
|
|
aggregatedMap.put(obtainedTriple, aggregated);
|
|
|
|
}
|
|
|
|
|
2019-11-26 11:50:31 +01:00
|
|
|
ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
|
|
|
|
sur.setServiceClass(serviceClass);
|
|
|
|
sur.setServiceName(serviceName);
|
|
|
|
sur.setCalledMethod(calledMethod);
|
|
|
|
sur.validate();
|
2020-03-31 16:16:05 +02:00
|
|
|
sur.validate();
|
2019-11-26 11:50:31 +01:00
|
|
|
// logger.trace("To Be aggregated ServiceUsageRecord {}", sur);
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
long oldMinInvocationTime = aggregated.getMinInvocationTime();
|
|
|
|
long oldMaxInvocationTime = aggregated.getMaxInvocationTime();
|
|
|
|
int oldOperationCount = aggregated.getOperationCount();
|
2019-11-26 11:50:31 +01:00
|
|
|
long oldDuration = aggregated.getDuration();
|
2020-01-07 11:07:06 +01:00
|
|
|
|
2019-11-26 11:50:31 +01:00
|
|
|
long surDuration = sur.getDuration();
|
|
|
|
|
2020-03-19 16:52:45 +01:00
|
|
|
try {
|
|
|
|
aggregated.aggregate(sur);
|
|
|
|
//logger.debug("Resulting Aggregated ServiceUsageRecord: {}", aggregated);
|
|
|
|
}catch(NotAggregatableRecordsExceptions e) {
|
|
|
|
logger.error("The Record is not aggregatable : {}", sur);
|
|
|
|
throw e;
|
|
|
|
}
|
2019-11-26 11:50:31 +01:00
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
long avgDuration = durationWeightedAverage(oldOperationCount, oldDuration, 1, surDuration);
|
2019-11-26 11:50:31 +01:00
|
|
|
Assert.assertTrue(aggregated.getDuration() == (avgDuration));
|
2020-01-07 11:07:06 +01:00
|
|
|
Assert.assertTrue(aggregated.getOperationCount() == ++oldOperationCount);
|
2019-11-26 11:50:31 +01:00
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
if(oldMinInvocationTime >= surDuration) {
|
2019-11-26 11:50:31 +01:00
|
|
|
Assert.assertTrue(aggregated.getMinInvocationTime() == surDuration);
|
2020-01-07 11:07:06 +01:00
|
|
|
} else {
|
|
|
|
Assert.assertTrue(aggregated.getMinInvocationTime() == oldMinInvocationTime);
|
2019-11-26 11:50:31 +01:00
|
|
|
}
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
if(oldMaxInvocationTime >= surDuration) {
|
|
|
|
Assert.assertTrue(aggregated.getMaxInvocationTime() == oldMaxInvocationTime);
|
|
|
|
} else {
|
2019-11-26 11:50:31 +01:00
|
|
|
Assert.assertTrue(aggregated.getMaxInvocationTime() == surDuration);
|
|
|
|
}
|
|
|
|
|
2019-11-15 14:40:11 +01:00
|
|
|
}
|
|
|
|
} catch(Exception e) {
|
|
|
|
throw e;
|
|
|
|
}
|
2019-11-25 17:33:44 +01:00
|
|
|
|
|
|
|
FilenameFilter filenameFilter = new FilenameFilter() {
|
|
|
|
@Override
|
|
|
|
public boolean accept(File dir, String name) {
|
|
|
|
boolean accept = name.endsWith(".csv");
|
2020-01-07 11:07:06 +01:00
|
|
|
return name.compareTo(requiredMatchesFileName) != 0 && accept;
|
2019-11-25 17:33:44 +01:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2020-01-08 16:07:13 +01:00
|
|
|
for(File elaborationFileNoMatch : rulesDirectory.listFiles(filenameFilter)) {
|
|
|
|
logger.info("Comparing examples which must not match from file {}", elaborationFileNoMatch.getName());
|
|
|
|
try(BufferedReader br = new BufferedReader(new FileReader(elaborationFileNoMatch))) {
|
|
|
|
for(String line; (line = br.readLine()) != null;) {
|
|
|
|
String[] splittedLine = line.split(",");
|
|
|
|
String serviceClass = splittedLine[0];
|
|
|
|
String serviceName = splittedLine[1];
|
|
|
|
String calledMethod = splittedLine[2];
|
|
|
|
Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod);
|
|
|
|
if(replace != null) {
|
|
|
|
logger.error("{} match {} but it should NOT. This MUST not occur.", line,
|
|
|
|
matcherReplace.getMultiMatcher().toString());
|
|
|
|
throw new Exception();
|
|
|
|
} else {
|
|
|
|
logger.trace("{} does NOT match as requested", line, replacementRegex.getServiceClass(),
|
|
|
|
replacementRegex.getServiceName(), replacementRegex.getCalledMethod());
|
|
|
|
}
|
|
|
|
|
|
|
|
ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
|
|
|
|
sur.setServiceClass(serviceClass);
|
|
|
|
sur.setServiceName(serviceName);
|
|
|
|
sur.setCalledMethod(calledMethod);
|
|
|
|
sur.validate();
|
|
|
|
//logger.trace("Should not be aggregated ServiceUsageRecord {}", sur);
|
|
|
|
for(AggregatedServiceUsageRecord asur : aggregatedMap.values()) {
|
|
|
|
try {
|
|
|
|
asur.aggregate(sur);
|
2020-03-05 18:03:17 +01:00
|
|
|
logger.error("The record {} has been aggregated and it should NOT", sur);
|
2020-01-08 16:07:13 +01:00
|
|
|
throw new Exception("The record has been aggregated and it should NOT");
|
|
|
|
} catch(NotAggregatableRecordsExceptions e) {
|
|
|
|
//logger.trace("{} is not aggragable as expected", sur);
|
2020-01-07 11:07:06 +01:00
|
|
|
}
|
2019-11-26 11:50:31 +01:00
|
|
|
}
|
2019-11-25 17:33:44 +01:00
|
|
|
}
|
2020-01-08 16:07:13 +01:00
|
|
|
} catch(Exception e) {
|
|
|
|
throw e;
|
2019-11-25 17:33:44 +01:00
|
|
|
}
|
|
|
|
}
|
2020-01-08 16:07:13 +01:00
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
logger.info(
|
|
|
|
"-----------------------------------------------------------------------------------------------------\n\n\n");
|
2019-11-15 14:40:11 +01:00
|
|
|
}
|
|
|
|
|
2020-03-31 16:16:05 +02:00
|
|
|
|
2019-11-15 14:40:11 +01:00
|
|
|
@Test
|
|
|
|
public void testSingleRule() throws Exception {
|
2020-01-07 11:07:06 +01:00
|
|
|
allRules();
|
2019-11-25 17:33:44 +01:00
|
|
|
File rulesDirectory = getRulesDirectory();
|
2020-04-10 15:19:55 +02:00
|
|
|
File rulesDirFile = new File(rulesDirectory, "3MConnector");
|
|
|
|
File ruleFile = new File(rulesDirFile, "3MConnector-connect.json");
|
2020-04-01 15:20:31 +02:00
|
|
|
testRule(ruleFile);
|
2019-11-15 14:40:11 +01:00
|
|
|
}
|
|
|
|
|
2020-04-01 15:20:31 +02:00
|
|
|
@Test
|
|
|
|
public void testAllRuleInAdirectory() throws Exception {
|
|
|
|
allRules();
|
|
|
|
File rulesDirectory = getRulesDirectory();
|
2020-04-10 15:19:55 +02:00
|
|
|
File rulesDirFile = new File(rulesDirectory, "3MConnector");
|
2020-04-01 15:20:31 +02:00
|
|
|
|
|
|
|
FilenameFilter filenameFilter = new FilenameFilter() {
|
|
|
|
@Override
|
|
|
|
public boolean accept(File dir, String name) {
|
|
|
|
return name.endsWith(".json");
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
List<File> rulesFiles = new ArrayList<>();
|
|
|
|
rulesFiles.addAll(Arrays.asList(rulesDirFile.listFiles(filenameFilter)));
|
|
|
|
|
|
|
|
for(File ruleFile : rulesFiles) {
|
|
|
|
testRule(ruleFile);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-01-07 11:07:06 +01:00
|
|
|
@Test
|
|
|
|
public void testAllRules() throws Exception {
|
|
|
|
List<File> rulesFiles = allRules();
|
|
|
|
|
|
|
|
for(File rulesFile : rulesFiles) {
|
|
|
|
testRule(rulesFile);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-23 12:33:07 +01:00
|
|
|
@Test
|
|
|
|
public void testMatcherReplace() throws Exception {
|
|
|
|
File rulesDirectory = getRulesDirectory();
|
2020-03-25 14:57:47 +01:00
|
|
|
File rulesDirFile = new File(rulesDirectory, "geoanalytics-gos");
|
|
|
|
File rulesFile = new File(rulesDirFile, "geoanalytics-gos-OTHER.json");
|
2020-03-23 12:33:07 +01:00
|
|
|
|
|
|
|
ObjectMapper mapper = DSMapper.getObjectMapper();
|
|
|
|
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
|
|
|
|
|
2020-03-25 14:57:47 +01:00
|
|
|
Replace replace = matcherReplace.check("geoanalytics","geoanalytics-gos-gos","/ShapeManagement/applyOnView");
|
2020-03-23 12:33:07 +01:00
|
|
|
|
|
|
|
logger.info("{}", replace);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-11-15 14:40:11 +01:00
|
|
|
}
|